@nshiab/simple-data-analysis-core 2.0.5 → 2.0.7

package/esm/methods/loadArray.d.ts

@@ -13,7 +13,7 @@ export declare function prepareArray(rows: {
     [key: string]: unknown;
 }[], columnTypes?: {
     [key: string]: string;
-}): PreparedArray;
+}, requireExplicitNestedTypes?: boolean): PreparedArray;
 export declare function executePreparedArray(simpleTable: SimpleTable, prepared: PreparedArray): Promise<void>;
 export {};
 //# sourceMappingURL=loadArray.d.ts.map

package/esm/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,GAC1C,aAAa,CA2Ff;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+BxB"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA4ExB"}

package/esm/methods/loadArray.js

@@ -1,4 +1,7 @@
 import { arrayValue, DuckDBDataChunk, DuckDBDateValue, DuckDBTimestampTZValue, DuckDBTimestampValue, DuckDBTimeValue, } from "@duckdb/node-api";
+import prepareGeometry from "../helpers/prepareGeometry.js";
+import prepareJSON from "../helpers/prepareJSON.js";
+import geometryFromJSON from "../helpers/geometryFromJSON.js";
 import queueOp from "../helpers/queueOp.js";
 import parseType from "../helpers/parseTypes.js";
 import parseDuckDBType from "../helpers/parseDuckDBType.js";
@@ -8,7 +11,7 @@ export default function loadArray(simpleTable, rows, options = {}) {
     if (rows.length === 0) {
         throw new Error("The array is empty. loadArray needs at least one object to infer the column types.");
     }
-    const prepared = prepareArray(rows, options.columnTypes);
+    const prepared = prepareArray(rows, options.columnTypes, true);
     queueOp(simpleTable, {
         kind: "barrier",
         method: "loadArray()",
@@ -16,17 +19,26 @@ export default function loadArray(simpleTable, rows, options = {}) {
         execute: () => executePreparedArray(simpleTable, prepared),
     });
 }
-export function prepareArray(rows, columnTypes = {}) {
+export function prepareArray(rows, columnTypes = {}, requireExplicitNestedTypes = false) {
     const keys = Object.keys(rows[0]);
     const overrides = new Map();
     for (const [key, type] of Object.entries(columnTypes)) {
         if (!keys.includes(key)) {
             throw new Error(`Unknown column ${JSON.stringify(key)} in loadArray columnTypes.`);
         }
+        if (type.toUpperCase() === "JSON") {
+            overrides.set(key, "JSON");
+            continue;
+        }
+        if (/^geometry/i.test(type)) {
+            if (!/^geometry\('EPSG:4326'\)$/i.test(type)) {
+                throw new Error(`Unsupported type ${JSON.stringify(type)} for column ${JSON.stringify(key)}. GeoJSON ingestion requires GEOMETRY('EPSG:4326'); reproject input to WGS84 first.`);
+            }
+            overrides.set(key, "GEOMETRY('EPSG:4326')");
+            continue;
+        }
         try {
-            const normalized = /^FLOAT\[[1-9]\d*\]$/i.test(type)
-                ? type.toUpperCase()
-                : parseType(type);
+            const normalized = parseType(type);
             parseDuckDBType(normalized);
             overrides.set(key, normalized);
         }
@@ -48,6 +60,14 @@ export function prepareArray(rows, columnTypes = {}) {
             }
             continue;
         }
+        if (requireExplicitNestedTypes) {
+            for (let j = 0; j < rows.length; j++) {
+                const cell = rows[j][key];
+                if (cell !== null && typeof cell === "object" && !(cell instanceof Date)) {
+                    throw new Error(`Column ${JSON.stringify(key)}, row ${j + 1}: array and object cells require an explicit columnTypes entry. Use JSON for nested data, FLOAT[n] for fixed-size vectors, or GEOMETRY('EPSG:4326') for GeoJSON geometries.`);
+                }
+            }
+        }
         const value = firstNonNullValue[i];
         const type = typeof value;
         if (type === "symbol" || type === "function") {
@@ -104,24 +124,59 @@ export async function executePreparedArray(simpleTable, prepared) {
         simpleTable.connection = simpleTable.sdb.connection;
     }
     const { keys, types, columnsData, rowCount } = prepared;
-    await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(simpleTable.name)}(${keys.map((key, i) => `${quoteIdentifier(key)} ${types[i]}`).join(", ")})`);
-    const appender = await simpleTable.connection
-        .createAppender(simpleTable.name);
-    const duckDBTypes = types.map((d) => parseDuckDBType(d));
-    // The maximum capacity of a DuckDB data chunk is 2048 rows.
-    const chunkSize = 2000;
-    for (let start = 0; start < rowCount; start += chunkSize) {
-        const end = Math.min(start + chunkSize, rowCount);
-        const dataChunk = DuckDBDataChunk.create(duckDBTypes, end - start);
-        for (let i = 0; i < keys.length; i++) {
-            dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-        }
-        appender.appendDataChunk(dataChunk);
+    const hasGeometry = types.includes("GEOMETRY('EPSG:4326')");
+    const needsConversion = hasGeometry || types.includes("JSON");
+    const staged = needsConversion
+        ? `__sda_array_${crypto.randomUUID().replaceAll("-", "")}`
+        : simpleTable.name;
+    const storageTypes = types.map((type) => type.startsWith("GEOMETRY") || type === "JSON" ? "VARCHAR" : type);
+    if (hasGeometry && !simpleTable.sdb.spatialLoaded) {
+        await simpleTable.sdb.customQuery("INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;");
+        simpleTable.sdb.spatialLoaded = true;
+    }
+    try {
+        await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${quoteIdentifier(staged)}(${keys.map((key, i) => `${quoteIdentifier(key)} ${storageTypes[i]}`).join(", ")})`);
+        const appender = await simpleTable.connection
+            .createAppender(staged);
+        try {
+            const duckDBTypes = storageTypes.map((d) => parseDuckDBType(d));
+            // The maximum capacity of a DuckDB data chunk is 2048 rows.
+            const chunkSize = 2000;
+            for (let start = 0; start < rowCount; start += chunkSize) {
+                const end = Math.min(start + chunkSize, rowCount);
+                const dataChunk = DuckDBDataChunk.create(duckDBTypes, end - start);
+                for (let i = 0; i < keys.length; i++) {
+                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
+                }
+                appender.appendDataChunk(dataChunk);
+            }
+            appender.flushSync();
+        }
+        finally {
+            appender.closeSync();
+        }
+        if (!needsConversion)
+            return;
+        // CTAS is atomic: conversion must finish before the destination is replaced.
+        await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(simpleTable.name)} AS SELECT ${keys.map((key, i) => types[i].startsWith("GEOMETRY")
+            ? `${geometryFromJSON(quoteIdentifier(key))} AS ${quoteIdentifier(key)}`
+            : types[i] === "JSON"
+                ? `${quoteIdentifier(key)}::JSON AS ${quoteIdentifier(key)}`
+                : quoteIdentifier(key)).join(", ")} FROM ${quoteIdentifier(staged)}`);
+    }
+    finally {
+        if (needsConversion) {
+            await simpleTable.sdb.customQuery(`DROP TABLE IF EXISTS ${quoteIdentifier(staged)}`);
+        }
     }
-    appender.flushSync();
 }
 /** Validates and snapshots a value for an explicitly declared destination type. */
 function prepareTypedValue(value, type, column, row) {
+    if (type === "JSON")
+        return prepareJSON(value, column, row);
+    if (type === "GEOMETRY('EPSG:4326')") {
+        return prepareGeometry(value, column, row);
+    }
     if (value === null || value === undefined || Number.isNaN(value))
         return null;
     let compatible = false;

package/esm/methods/loadData.js

@@ -47,6 +47,9 @@ export function loadDataQuery(table, files, options = {}) {
     return `CREATE OR REPLACE TABLE ${quoteIdentifier(table)} AS ${loadDataSelect(files, options)};`;
 }
 function loadDataSelect(files, options = {}) {
+    if (Object.values(options.columnTypes ?? {}).some((type) => /^\s*geometry\b/i.test(type))) {
+        throw new Error("Geometry column types are not supported by loadData(). Use loadGeoData() instead.");
+    }
     const fileExtension = getExtension(files[0]);
     if (options.fileType !== undefined &&
         !["csv", "dsv", "json", "parquet", "excel"].includes(options.fileType)) {

package/esm/methods/log.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function log(simpleTable: SimpleTable, options?: "all" | number | {
+    count?: number | "all";
+    types?: boolean;
+    conditions?: string;
+}): Promise<void>;
+//# sourceMappingURL=log.d.ts.map

package/esm/methods/log.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../../src/methods/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,GAAG,CAC/B,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,KAAK,GAAG,MAAM,GAAG;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,iBA4EP"}

package/esm/methods/log.js

@@ -0,0 +1,63 @@
+import logData from "../helpers/logData.js";
+import getData from "./getData.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import formatNumber from "../helpers/formatNumber.js";
+export default async function log(simpleTable, options = {}) {
+    if (simpleTable.connection === undefined) {
+        await simpleTable.sdb.start();
+        simpleTable.db = simpleTable.sdb.db;
+        simpleTable.connection = simpleTable.sdb.connection;
+    }
+    if (simpleTable.connection === undefined) {
+        throw new Error("simpleTable.connection is undefined");
+    }
+    let count;
+    if (typeof options === "number") {
+        count = options;
+    }
+    else if (options === "all") {
+        count = await simpleTable.getRowCount();
+    }
+    else if (typeof options === "object") {
+        if (options.count === "all") {
+            count = await simpleTable.getRowCount();
+        }
+        else if (typeof options.count === "number") {
+            count = options.count;
+        }
+        else {
+            count = simpleTable.rowsToLog;
+        }
+    }
+    else {
+        count = simpleTable.rowsToLog;
+    }
+    const types = typeof options === "object"
+        ? options.types ?? simpleTable.typesToLog
+        : simpleTable.typesToLog;
+    const conditions = typeof options === "object"
+        ? options.conditions ?? undefined
+        : undefined;
+    if (simpleTable.connection === undefined ||
+        !(await simpleTable.sdb.hasTable(simpleTable.name))) {
+        console.log(`\nTable ${simpleTable.name}: no data`);
+    }
+    else {
+        console.log(`\nTable ${simpleTable.name}:`);
+        conditions && console.log(`Conditions: ${conditions}`);
+        const data = await getData(simpleTable, { limit: count, conditions }, "log()");
+        logData(types ? await simpleTable.getTypes() : null, data, simpleTable.charsToLog);
+        const rowCount = conditions
+            ? parseInt((await simpleTable.sdb.customQuery(`select count(*) as count from ${quoteIdentifier(simpleTable.name)} where ${conditions}`, { returnData: true }))[0].count)
+            : await simpleTable.getRowCount();
+        const displayedRows = data.length;
+        const showing = displayedRows < rowCount
+            ? ` / showing ${formatNumber(displayedRows)} rows`
+            : "";
+        const charsToLog = typeof simpleTable.charsToLog === "number"
+            ? ` (charsToLog: ${simpleTable.charsToLog})`
+            : "";
+        console.log(`${formatNumber(rowCount)} rows in total${showing}${charsToLog}`);
+    }
+    return;
+}

package/esm/methods/logBottom.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function logBottom(simpleTable: SimpleTable, count?: number, options?: {
+    originalOrder?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=logBottom.d.ts.map

package/esm/methods/logBottom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logBottom.d.ts","sourceRoot":"","sources":["../../src/methods/logBottom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,SAAS,CACrC,WAAW,EAAE,WAAW,EACxB,KAAK,CAAC,EAAE,MAAM,EACd,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAO,iBAW1C"}

package/esm/methods/logBottom.js

@@ -0,0 +1,9 @@
+import logData from "../helpers/logData.js";
+import getBottom from "./getBottom.js";
+export default async function logBottom(simpleTable, count, options = {}) {
+    const rows = count ?? simpleTable.rowsToLog;
+    console.log(`\nTable ${simpleTable.name} (${rows} bottom rows):`);
+    const data = await getBottom(simpleTable, rows, options, "logBottom()");
+    logData(null, data, simpleTable.charsToLog);
+    return;
+}

package/esm/methods/removeTable.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"removeTable.d.ts","sourceRoot":"","sources":["../../src/methods/removeTable.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,iBAgBzB"}
+{"version":3,"file":"removeTable.d.ts","sourceRoot":"","sources":["../../src/methods/removeTable.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,iBAiBzB"}

package/esm/methods/removeTable.js

@@ -1,8 +1,10 @@
+import { assertCacheTableMutation } from "../helpers/cacheTableDependencies.js";
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
 import { retainRegisteredTables } from "../helpers/tableRegistry.js";
 export default async function removeTable(simpleTable) {
+    assertCacheTableMutation(simpleTable);
     await queryDB(simpleTable, `DROP TABLE ${quoteIdentifier(simpleTable.name)};`, mergeOptions(simpleTable, {
         table: null,
         method: "removeTable()",

package/esm/methods/removeTables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"removeTables.d.ts","sourceRoot":"","sources":["../../src/methods/removeTables.ts"],"names":[],"mappings":"AAGA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAIjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBAkCxD"}
+{"version":3,"file":"removeTables.d.ts","sourceRoot":"","sources":["../../src/methods/removeTables.ts"],"names":[],"mappings":"AAIA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAIjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBA2CxD"}

package/esm/methods/removeTables.js

@@ -1,3 +1,4 @@
+import { assertCacheTableMutation } from "../helpers/cacheTableDependencies.js";
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
@@ -11,6 +12,12 @@ export default async function removeTables(simpleDB, tables) {
             ? tables
             : [tables];
     assertSameDatabase(simpleDB, tablesToBeRemoved.filter((table) => table instanceof SimpleTable), "removeTables()");
+    const names = tablesToBeRemoved.map((table) => (table instanceof SimpleTable ? table.name : table).toLowerCase());
+    for (const table of simpleDB.getTables()) {
+        if (names.includes(table.name.toLowerCase())) {
+            assertCacheTableMutation(table);
+        }
+    }
     await queryDB(simpleDB, tablesToBeRemoved.map((d) => `DROP TABLE ${quoteIdentifier(d instanceof SimpleTable ? d.name : d)};`).join("\n"), mergeOptions(simpleDB, {
         table: null,
         method: "removeTable()",

package/esm/methods/renameTable.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"renameTable.d.ts","sourceRoot":"","sources":["../../src/methods/renameTable.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,iBAcb"}
+{"version":3,"file":"renameTable.d.ts","sourceRoot":"","sources":["../../src/methods/renameTable.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,iBAgBb"}

package/esm/methods/renameTable.js

@@ -1,8 +1,12 @@
+import { assertCacheTableMutation } from "../helpers/cacheTableDependencies.js";
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
 import { markTableChanged } from "../helpers/tableGeneration.js";
+import { assertTableNameAvailable } from "../helpers/tableRegistry.js";
 export default async function renameTable(simpleTable, name) {
+    assertCacheTableMutation(simpleTable);
+    assertTableNameAvailable(simpleTable.sdb, name, simpleTable);
     await queryDB(simpleTable, `ALTER TABLE ${quoteIdentifier(simpleTable.name)} RENAME TO ${quoteIdentifier(name)};`, mergeOptions(simpleTable, {
         table: simpleTable.name,
         method: "renameTable()",

package/esm/methods/rowToText.js

@@ -19,8 +19,8 @@ export default function rowToText(simpleTable, columns, newColumn) {
                     throw new Error(`The column ${col} is of type ${allTypes[col]}. The rowToText() method only works with string columns. Please convert the column to string first with the .convert() method.`);
                 }
             }
-            const parts = columns.map((col) => `? || COALESCE(${quoteIdentifier(col)}, ?)`);
-            const concatenatedExpression = parts.join(" || ");
+            const parts = columns.map((col) => `concat(?, COALESCE(${quoteIdentifier(col)}, ?))`);
+            const concatenatedExpression = `concat(${parts.join(", ")})`;
             return `SELECT *, CAST(${concatenatedExpression} AS VARCHAR) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
         },
     });

package/esm/methods/selectTables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selectTables.d.ts","sourceRoot":"","sources":["../../src/methods/selectTables.ts"],"names":[],"mappings":"AAGA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAKjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBA+CxD"}
+{"version":3,"file":"selectTables.d.ts","sourceRoot":"","sources":["../../src/methods/selectTables.ts"],"names":[],"mappings":"AAIA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAKjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBAqDxD"}

package/esm/methods/selectTables.js

@@ -1,3 +1,4 @@
+import { assertCacheTableMutation } from "../helpers/cacheTableDependencies.js";
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
@@ -9,6 +10,11 @@ export default async function selectTables(simpleDB, tables) {
     const selected = Array.isArray(tables) ? tables : [tables];
     assertSameDatabase(simpleDB, selected.filter((table) => table instanceof SimpleTable), "selectTables()");
     const tablesToBeSelected = selected.map((t) => t instanceof SimpleTable ? t.name : t);
+    for (const table of simpleDB.getTables()) {
+        if (!tablesToBeSelected.includes(table.name)) {
+            assertCacheTableMutation(table);
+        }
+    }
     const existingTables = await simpleDB.getTableNames();
     const missingTables = tablesToBeSelected.filter((table) => !existingTables.includes(table));
     if (missingTables.length > 0) {

package/esm/methods/setTypes.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function setTypes(simpleTable: SimpleTable, types: {
-    [key: string]: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | `geometry('${string}')` | `GEOMETRY('${string}')`;
+    [key: string]: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | `float[${number}]` | `FLOAT[${number}]` | `geometry('${string}')` | `GEOMETRY('${string}')`;
 }): void;
 //# sourceMappingURL=setTypes.d.ts.map

package/esm/methods/setTypes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"setTypes.d.ts","sourceRoot":"","sources":["../../src/methods/setTypes.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,WAAW,EAAE,WAAW,EACxB,KAAK,EAAE;IACL,CAAC,GAAG,EAAE,MAAM,GACR,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,aAAa,MAAM,IAAI,GACvB,aAAa,MAAM,IAAI,CAAC;CAC7B,QAkCF"}
+{"version":3,"file":"setTypes.d.ts","sourceRoot":"","sources":["../../src/methods/setTypes.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,WAAW,EAAE,WAAW,EACxB,KAAK,EAAE;IACL,CAAC,GAAG,EAAE,MAAM,GACR,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,MAAM,GACN,MAAM,GACN,SAAS,MAAM,GAAG,GAClB,SAAS,MAAM,GAAG,GAClB,aAAa,MAAM,IAAI,GACvB,aAAa,MAAM,IAAI,CAAC;CAC7B,QAkCF"}

package/esm/methods/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/methods/stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA+B,MAAM,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IACP,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,GACL,cAAc,CAAC;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CAmE7D"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/methods/stream.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA+B,MAAM,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IACP,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,GACL,cAAc,CAAC;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CAsE7D"}

package/esm/methods/stream.js

@@ -1,3 +1,4 @@
+import cleanSQL from "../helpers/cleanSQL.js";
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import { makeConverter } from "../helpers/runQuery.js";
 import SDAError from "../class/SDAError.js";
@@ -17,7 +18,7 @@ export default async function* stream(simpleTable, options = {}) {
             ? [options.columns]
             : options.columns)
         : undefined;
-    const query = `SELECT ${columns ? columns.map((d) => `${quoteIdentifier(d)}`).join(", ") : "*"} FROM ${quoteIdentifier(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""};`;
+    const query = cleanSQL(`SELECT ${columns ? columns.map((d) => `${quoteIdentifier(d)}`).join(", ") : "*"} FROM ${quoteIdentifier(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""};`, simpleTable.sdb.expressionSyntax);
     try {
         await observeQuery(simpleTable.connection, query, [], {
             logSQL: simpleTable.sdb.logSQL,

package/esm/methods/updateWithJS.d.ts

@@ -3,9 +3,7 @@ type Row = {
     [key: string]: unknown;
 };
 type DataModifier = (rows: Row[]) => Row[] | Promise<Row[]>;
-type UpdateWithJSOptions = {
-    batchSize?: number;
-};
+type UpdateWithJSOptions = NonNullable<Parameters<SimpleTable["updateWithJS"]>[1]>;
 export default function updateWithJS(simpleTable: SimpleTable, dataModifier: DataModifier, options?: UpdateWithJSOptions): void;
 export {};
 //# sourceMappingURL=updateWithJS.d.ts.map

package/esm/methods/updateWithJS.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"updateWithJS.d.ts","sourceRoot":"","sources":["../../src/methods/updateWithJS.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,KAAK,GAAG,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEtC,KAAK,YAAY,GAAG,CAClB,IAAI,EAAE,GAAG,EAAE,KACR,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAE5B,KAAK,mBAAmB,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,WAAW,EAAE,WAAW,EACxB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,mBAAwB,GAChC,IAAI,CAeN"}
+{"version":3,"file":"updateWithJS.d.ts","sourceRoot":"","sources":["../../src/methods/updateWithJS.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAQvD,KAAK,GAAG,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEtC,KAAK,YAAY,GAAG,CAClB,IAAI,EAAE,GAAG,EAAE,KACR,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAE5B,KAAK,mBAAmB,GAAG,WAAW,CACpC,UAAU,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3C,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,WAAW,EAAE,WAAW,EACxB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,mBAAwB,GAChC,IAAI,CAeN"}

package/esm/methods/updateWithJS.js

@@ -1,7 +1,12 @@
+import prepareGeometry from "../helpers/prepareGeometry.js";
+import geometryFromJSON from "../helpers/geometryFromJSON.js";
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import { retainRegisteredTables } from "../helpers/tableRegistry.js";
 import queueOp from "../helpers/queueOp.js";
-import { executePreparedArray, prepareArray } from "./loadArray.js";
+import { prepareArray } from "./loadArray.js";
+import { DuckDBDataChunk } from "@duckdb/node-api";
+import readMutationRows from "../helpers/readMutationRows.js";
+import parseDuckDBType from "../helpers/parseDuckDBType.js";
 export default function updateWithJS(simpleTable, dataModifier, options = {}) {
     if (options.batchSize !== undefined &&
         (!Number.isInteger(options.batchSize) || options.batchSize < 1)) {
@@ -17,39 +22,35 @@ export default function updateWithJS(simpleTable, dataModifier, options = {}) {
 }
 async function executeUpdateWithJS(simpleTable, dataModifier, options) {
     const types = await simpleTable.getTypes();
-    // Geometry columns are typed as GEOMETRY('<crs>'), not a bare "GEOMETRY".
-    if (Object.values(types).some((d) => d.toUpperCase().startsWith("GEOMETRY"))) {
-        throw new Error("updateWithJS doesn't work with tables containing geometries.");
-    }
-    if (options.batchSize === undefined) {
-        const oldData = await simpleTable.getData();
-        if (!oldData) {
-            throw new Error("No data from getData.");
+    const geometryColumns = Object.keys(types).filter((key) => types[key].toUpperCase().startsWith("GEOMETRY"));
+    for (const column of geometryColumns) {
+        if (types[column].toUpperCase() !== "GEOMETRY('EPSG:4326')") {
+            throw new Error(`Column ${JSON.stringify(column)} has CRS ${types[column]}. Call .reproject("EPSG:4326", { column: ${JSON.stringify(column)} }) before updateWithJS().`);
         }
-        const newData = await dataModifier(oldData);
-        if (newData.length === 0) {
-            if (oldData.length === 0) {
-                // Empty table in, empty data out: nothing to update.
-                return;
-            }
-            throw new Error("The dataModifier returned no rows. updateWithJS can't infer the table schema from zero rows.");
+        const name = quoteIdentifier(column);
+        // Compare the actual binary representation, including dimensions and double
+        // precision, rather than assuming GeoJSON serialization is lossless.
+        const result = await simpleTable.connection.runAndReadAll(`SELECT count(*) AS invalid FROM ${quoteIdentifier(simpleTable.name)}
+       WHERE ${name} IS NOT NULL AND
+       ST_AsWKB(${name}) != ST_AsWKB(${geometryFromJSON(`ST_AsGeoJSON(${name})`)})`);
+        if (result.getRows()[0][0] !== 0n) {
+            throw new Error(`Column ${JSON.stringify(column)} contains geometry that cannot round-trip through GeoJSON without losing precision or dimensions; unsupported representations cannot be updated with updateWithJS().`);
         }
-        await executePreparedArray(simpleTable, prepareArray(newData));
-        return;
     }
-    const batchSize = options.batchSize;
-    if (Object.keys(types).includes("__sda_rowid")) {
-        throw new Error('The table has a column named "__sda_rowid", which conflicts with the internal column used by the batchSize option. Rename it or run updateWithJS without batchSize.');
+    for (const key of Object.keys(options.columnTypes ?? {})) {
+        if (Object.keys(types).some((name) => name.toLowerCase() === key.toLowerCase())) {
+            throw new Error(`updateWithJS() columnTypes may only declare new columns; ${JSON.stringify(key)} already exists.`);
+        }
     }
-    const rowCount = await simpleTable.getRowCount();
-    if (rowCount === 0) {
-        // Same behavior as the non-batched path on an empty table.
-        const newData = await dataModifier([]);
-        if (newData.length === 0) {
-            return;
+    const batchSize = options.batchSize;
+    const geometryBatch = batchSize !== undefined && geometryColumns.length > 0;
+    if (batchSize !== undefined) {
+        // DuckDB resolves identifiers case-insensitively, and a real rowid column
+        // shadows its hidden row identifier used for pagination.
+        const conflictingColumn = Object.keys(types).find((column) => ["rowid", "__sda_rowid"].includes(column.toLowerCase()));
+        if (conflictingColumn !== undefined) {
+            throw new Error(`The table has a column named "${conflictingColumn}", which conflicts with the internal column used by the batchSize option. Rename it or run updateWithJS without batchSize.`);
         }
-        await executePreparedArray(simpleTable, prepareArray(newData));
-        return;
     }
     // Rows are pulled in batches by rowid, passed through the modifier, and
     // accumulated in a temporary table, so only one batch of rows is
@@ -60,27 +61,112 @@ async function executeUpdateWithJS(simpleTable, dataModifier, options) {
     try {
         let first = true;
         let lastRowid = null;
+        let sawRows = false;
+        let outputOffset = 0;
+        let inputOffset = 0;
         while (true) {
-            const batch = (await simpleTable.sdb.customQuery(`SELECT *, rowid AS __sda_rowid FROM ${quoteIdentifier(simpleTable.name)}${lastRowid === null ? "" : ` WHERE rowid > ${lastRowid}`} ORDER BY rowid LIMIT ${batchSize}`, { returnData: true }));
-            if (batch.length === 0) {
+            const page = `${quoteIdentifier(simpleTable.name)}${lastRowid === null ? "" : ` WHERE rowid > ${lastRowid}`}${batchSize === undefined ? "" : ` ORDER BY rowid LIMIT ${batchSize}`}`;
+            // Select the batch before serializing geometry. Otherwise DuckDB can
+            // evaluate ST_AsGeoJSON on remaining rows before applying the LIMIT,
+            // repeatedly converting polygons that belong to later batches.
+            const from = geometryBatch
+                ? `(SELECT *, rowid AS __sda_rowid FROM ${page}) AS __sda_batch ORDER BY __sda_rowid`
+                : page;
+            const source = await readMutationRows(simpleTable.connection, `SELECT ${Object.keys(types).map((key) => geometryColumns.includes(key)
+                ? `ST_AsGeoJSON(${quoteIdentifier(key)})::VARCHAR AS ${quoteIdentifier(key)}`
+                : quoteIdentifier(key)).join(", ")}${batchSize === undefined
+                ? ""
+                : geometryBatch
+                    ? ", __sda_rowid"
+                    : ", rowid AS __sda_rowid"} FROM ${from}`, batchSize === undefined ? undefined : "__sda_rowid");
+            const batch = source.rows;
+            const inputCount = batch.length;
+            if (inputCount === 0 && sawRows)
                 break;
+            sawRows ||= inputCount > 0;
+            if (batchSize !== undefined && batch.length > 0) {
+                // Keep the cursor separate: adding and deleting it on every row also
+                // makes ordinary callback operations such as object spread slower.
+                lastRowid = source.lastCursorValue;
             }
-            lastRowid = batch[batch.length - 1].__sda_rowid;
-            for (const row of batch) {
-                delete row.__sda_rowid;
+            for (const [index, row] of batch.entries()) {
+                for (const key of geometryColumns) {
+                    row[key] = row[key] === null ? null : JSON.parse(row[key]);
+                    prepareGeometry(row[key], key, inputOffset + index + 1);
+                }
             }
+            inputOffset += inputCount;
             const modified = await dataModifier(batch);
             if (modified.length === 0) {
+                if (!sawRows)
+                    return;
+                if (batchSize === undefined)
+                    break;
                 continue;
             }
-            await executePreparedArray(scratch, prepareArray(modified));
+            const keys = Object.keys(modified[0]);
+            for (const key of Object.keys(options.columnTypes ?? {})) {
+                if (!keys.includes(key)) {
+                    throw new Error(`Unknown column ${JSON.stringify(key)} in updateWithJS columnTypes.`);
+                }
+            }
+            const newKeys = keys.filter((key) => !source.types.has(key));
+            const added = newKeys.length === 0 ? undefined : prepareArray(modified.map((row) => Object.fromEntries(newKeys.map((key) => [key, row[key]]))), options.columnTypes);
+            const outputGeometry = keys.filter((key) => geometryColumns.includes(key) ||
+                (added && added.types[newKeys.indexOf(key)] === "GEOMETRY('EPSG:4326')"));
+            const outputJSON = keys.filter((key) => types[key] === "JSON" ||
+                (added && added.types[newKeys.indexOf(key)] === "JSON"));
+            if (outputGeometry.length && !simpleTable.sdb.spatialLoaded) {
+                await simpleTable.sdb.customQuery("INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;");
+                simpleTable.sdb.spatialLoaded = true;
+            }
+            const geometryData = new Map(outputGeometry.map((key) => [
+                key,
+                geometryColumns.includes(key)
+                    ? modified.map((row, i) => prepareGeometry(row[key], key, outputOffset + i + 1))
+                    : added.columnsData[newKeys.indexOf(key)],
+            ]));
+            outputOffset += modified.length;
+            const columnTypes = keys.map((key) => outputGeometry.includes(key) || outputJSON.includes(key)
+                ? parseDuckDBType("VARCHAR")
+                : source.types.get(key) ??
+                    parseDuckDBType(added.types[newKeys.indexOf(key)]));
+            await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(scratch.name)} (${keys.map((key, i) => `${quoteIdentifier(key)} ${columnTypes[i]}`)
+                .join(", ")})`);
+            const appender = await simpleTable.connection.createAppender(scratch.name);
+            try {
+                for (let start = 0; start < modified.length; start += 2000) {
+                    const end = Math.min(start + 2000, modified.length);
+                    const chunk = DuckDBDataChunk.create(columnTypes, end - start);
+                    for (let i = 0; i < keys.length; i++) {
+                        const key = keys[i];
+                        chunk.setColumnValues(i, outputGeometry.includes(key)
+                            ? geometryData.get(key).slice(start, end)
+                            : source.types.has(key)
+                                ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[i]))
+                                : added.columnsData[newKeys.indexOf(key)].slice(start, end));
+                    }
+                    appender.appendDataChunk(chunk);
+                }
+                appender.flushSync();
+            }
+            finally {
+                appender.closeSync();
+            }
+            const projection = keys.map((key) => outputGeometry.includes(key)
+                ? `${geometryFromJSON(quoteIdentifier(key))} AS ${quoteIdentifier(key)}`
+                : outputJSON.includes(key)
+                    ? `${quoteIdentifier(key)}::JSON AS ${quoteIdentifier(key)}`
+                    : quoteIdentifier(key)).join(", ");
             if (first) {
-                await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(accumulator)} AS SELECT * FROM ${quoteIdentifier(scratch.name)}`);
+                await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(accumulator)} AS SELECT ${projection} FROM ${quoteIdentifier(scratch.name)}`);
                 first = false;
             }
             else {
-                await simpleTable.sdb.customQuery(`INSERT INTO ${quoteIdentifier(accumulator)} BY NAME SELECT * FROM ${quoteIdentifier(scratch.name)}`);
+                await simpleTable.sdb.customQuery(`INSERT INTO ${quoteIdentifier(accumulator)} BY NAME SELECT ${projection} FROM ${quoteIdentifier(scratch.name)}`);
             }
+            if (batchSize === undefined || inputCount === 0)
+                break;
         }
         if (first) {
             throw new Error("The dataModifier returned no rows. updateWithJS can't infer the table schema from zero rows.");