@nshiab/simple-data-analysis-core 2.0.4 → 2.0.6

package/script/methods/getBottom.js

@@ -7,14 +7,14 @@ exports.default = getBottom;
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
-async function getBottom(simpleTable, count, options = {}) {
+async function getBottom(simpleTable, count, options = {}, method = "getBottom()") {
     const queryResult = await (0, queryDB_js_1.default)(simpleTable, `WITH "numberedRowsForGetBottom" AS (
                 SELECT *, row_number() OVER () as "rowNumberForGetBottom" FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""}
             )
             SELECT * FROM "numberedRowsForGetBottom" ORDER BY "rowNumberForGetBottom" DESC LIMIT ${count};`, (0, mergeOptions_js_1.default)(simpleTable, {
         table: simpleTable.name,
         returnData: true,
-        method: "getBottom()",
+        method,
         parameters: { count, options },
     }));
     if (!queryResult) {

package/script/methods/getData.d.ts

@@ -3,7 +3,7 @@ export default function getData(simpleTable: SimpleTable, options?: {
     columns?: string | string[];
     conditions?: string;
     limit?: number;
-}): Promise<{
+}, method?: "getData()" | "log()"): Promise<{
     [key: string]: unknown;
 }[]>;
 //# sourceMappingURL=getData.d.ts.map

package/script/methods/getData.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"getData.d.ts","sourceRoot":"","sources":["../../src/methods/getData.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,OAAO,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IACP,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX;;KAmCP"}
+{"version":3,"file":"getData.d.ts","sourceRoot":"","sources":["../../src/methods/getData.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,OAAO,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IACP,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,EACN,MAAM,GAAE,WAAW,GAAG,OAAqB;;KAmC5C"}

package/script/methods/getData.js

@@ -8,7 +8,7 @@ const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"))
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const quoteQualifiedIdentifier_js_1 = __importDefault(require("../helpers/quoteQualifiedIdentifier.js"));
-async function getData(simpleTable, options = {}) {
+async function getData(simpleTable, options = {}, method = "getData()") {
     if (options.limit !== undefined &&
         (!Number.isInteger(options.limit) || options.limit < 0)) {
         throw new Error("getData() limit must be an integer greater than or equal to 0.");
@@ -23,7 +23,7 @@ async function getData(simpleTable, options = {}) {
         : "*"} from ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""}${options.limit === undefined ? "" : ` LIMIT ${options.limit}`}`, (0, mergeOptions_js_1.default)(simpleTable, {
         returnData: true,
         table: simpleTable.name,
-        method: "getData()",
+        method,
         parameters: { options },
     })));
 }

package/script/methods/join.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../src/methods/join.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAOvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,GACL,WAAW,CAiBb"}
+{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../src/methods/join.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,GACL,WAAW,CAkBb"}

package/script/methods/join.js

@@ -6,8 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = join;
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const getIdenticalColumns_js_1 = __importDefault(require("../helpers/getIdenticalColumns.js"));
-const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
-const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const assertSameDatabase_js_1 = __importDefault(require("../helpers/assertSameDatabase.js"));
 function join(leftTable, rightTable, options = {}) {
@@ -19,14 +17,15 @@ function join(leftTable, rightTable, options = {}) {
         ? leftTable.sdb.newTable(options.outputTable)
         : leftTable;
     (0, queueOp_js_1.default)(outputTable, {
-        kind: "barrier",
+        kind: "source",
+        rawSQL: [(0, quoteIdentifier_js_1.default)(leftTable.name), (0, quoteIdentifier_js_1.default)(rightTable.name)],
         method: "join()",
         parameters: { rightTable: rightTable.name, options },
-        execute: () => executeJoin(leftTable, rightTable, outputTable, options),
+        buildSelect: () => buildJoinSelect(leftTable, rightTable, options),
     });
     return outputTable;
 }
-async function executeJoin(leftTable, rightTable, outputTable, options) {
+async function buildJoinSelect(leftTable, rightTable, options) {
     const leftTableColumns = await leftTable.getColumns();
     const rightTableColumns = await rightTable.getColumns();
     const identicalColumns = (0, getIdenticalColumns_js_1.default)(leftTableColumns, rightTableColumns);
@@ -80,17 +79,10 @@ async function executeJoin(leftTable, rightTable, outputTable, options) {
             .filter((d) => !on.includes(d))
             .map((d) => `${(0, quoteIdentifier_js_1.default)(rightTable.name)}.${(0, quoteIdentifier_js_1.default)(d)}`),
     ].join(", ");
-    await (0, queryDB_js_1.default)(leftTable, joinQuery(leftTable.name, rightTable.name, on, type, outputTable.name, selectList), (0, mergeOptions_js_1.default)(leftTable, {
-        table: outputTable.name,
-        method: "join()",
-        parameters: {
-            rightTable: rightTable.name,
-            options,
-        },
-    }));
+    return joinQuery(leftTable.name, rightTable.name, on, type, selectList);
 }
-function joinQuery(leftTable, rightTable, on, join, outputTable, selectList) {
-    let query = `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable)} AS SELECT ${selectList}`;
+function joinQuery(leftTable, rightTable, on, join, selectList) {
+    let query = `SELECT ${selectList}`;
     if (join === "inner") {
         query += ` FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)}`;
     }
@@ -107,6 +99,6 @@ function joinQuery(leftTable, rightTable, on, join, outputTable, selectList) {
         throw new Error(`Unknown ${join} join.`);
     }
     query += ` ON (${on.map((d) => `${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(d)} = ${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(d)}`)
-        .join(" AND ")});\n`;
+        .join(" AND ")})`;
     return query;
 }

package/script/methods/joinGeo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"joinGeo.d.ts","sourceRoot":"","sources":["../../src/methods/joinGeo.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAQvD,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EACjD,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;IAClD,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,GACL,WAAW,CAmBb"}
+{"version":3,"file":"joinGeo.d.ts","sourceRoot":"","sources":["../../src/methods/joinGeo.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EACjD,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;IAClD,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,GACL,WAAW,CAqBb"}

package/script/methods/joinGeo.js

@@ -8,8 +8,6 @@ const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier
 const capitalize_js_1 = __importDefault(require("../helpers/capitalize.js"));
 const findGeoColumnFromSchema_js_1 = __importDefault(require("../helpers/findGeoColumnFromSchema.js"));
 const getIdenticalColumns_js_1 = __importDefault(require("../helpers/getIdenticalColumns.js"));
-const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
-const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const assertSameDatabase_js_1 = __importDefault(require("../helpers/assertSameDatabase.js"));
 function joinGeo(leftTable, method, rightTable, options = {}) {
@@ -22,10 +20,12 @@ function joinGeo(leftTable, method, rightTable, options = {}) {
         ? leftTable.sdb.newTable(options.outputTable)
         : leftTable;
     (0, queueOp_js_1.default)(outputTable, {
-        kind: "barrier",
+        kind: "source",
+        rawSQL: [(0, quoteIdentifier_js_1.default)(leftTable.name), (0, quoteIdentifier_js_1.default)(rightTable.name)],
+        needsSpatial: true,
         method: "joinGeo()",
         parameters: { method, rightTable: rightTable.name, options },
-        execute: () => executeJoinGeo(leftTable, method, rightTable, outputTable, options),
+        buildSelect: () => buildJoinGeoSelect(leftTable, method, rightTable, options),
     });
     return outputTable;
 }
@@ -49,7 +49,7 @@ function validateJoinGeoOptions(method, options) {
 function formatReceivedValue(value) {
     return JSON.stringify(value) ?? String(value);
 }
-async function executeJoinGeo(leftTable, method, rightTable, outputTable, options) {
+async function buildJoinGeoSelect(leftTable, method, rightTable, options) {
     const leftTypes = await leftTable.getTypes();
     const rightTypes = await rightTable.getTypes();
     const leftColumn = options.leftColumn ??
@@ -89,19 +89,10 @@ async function executeJoinGeo(leftTable, method, rightTable, outputTable, option
         ]),
     ].join(", ");
     const type = options.type ?? "left";
-    await (0, queryDB_js_1.default)(leftTable, joinGeoQuery(leftTable.name, leftColumn, method, rightTable.name, rightColumn, type, outputTable.name, selectList, options.distance, options.distanceMethod), (0, mergeOptions_js_1.default)(leftTable, {
-        table: outputTable.name,
-        method: "joinGeo()",
-        parameters: {
-            leftTable: leftTable.name,
-            method,
-            rightTable: rightTable.name,
-            options,
-        },
-    }));
+    return joinGeoQuery(leftTable.name, leftColumn, method, rightTable.name, rightColumn, type, selectList, options.distance, options.distanceMethod);
 }
-function joinGeoQuery(leftTable, leftColumn, method, rightTable, rightColumn, join, outputTable, selectList, distance, distanceMethod) {
-    let query = `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable)} AS SELECT ${selectList}`;
+function joinGeoQuery(leftTable, leftColumn, method, rightTable, rightColumn, join, selectList, distance, distanceMethod) {
+    let query = `SELECT ${selectList}`;
     if (join === "inner") {
         query += ` FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)}`;
     }
@@ -118,11 +109,11 @@ function joinGeoQuery(leftTable, leftColumn, method, rightTable, rightColumn, jo
         throw new Error(`Unknown ${join} join.`);
     }
     if (method === "intersect") {
-        query += ` ON ST_Intersects(${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(leftColumn)}, ${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(rightColumn)});`;
+        query += ` ON ST_Intersects(${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(leftColumn)}, ${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(rightColumn)})`;
     }
     else if (method === "inside") {
         // Order is important
-        query += ` ON ST_Covers(${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(rightColumn)}, ${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(leftColumn)});`;
+        query += ` ON ST_Covers(${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(rightColumn)}, ${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(leftColumn)})`;
     }
     else if (method === "withinDistance") {
         if (typeof distance === "number") {

package/script/methods/loadArray.d.ts

@@ -2,7 +2,7 @@ import { type DuckDBValue } from "@duckdb/node-api";
 import type SimpleTable from "../class/SimpleTable.js";
 export default function loadArray(simpleTable: SimpleTable, rows: {
     [key: string]: unknown;
-}[]): void;
+}[], options?: NonNullable<Parameters<SimpleTable["loadArray"]>[1]>): void;
 type PreparedArray = {
     keys: string[];
     types: string[];
@@ -11,7 +11,9 @@ type PreparedArray = {
 };
 export declare function prepareArray(rows: {
     [key: string]: unknown;
-}[]): PreparedArray;
+}[], columnTypes?: {
+    [key: string]: string;
+}, requireExplicitNestedTypes?: boolean): PreparedArray;
 export declare function executePreparedArray(simpleTable: SimpleTable, prepared: PreparedArray): Promise<void>;
 export {};
 //# sourceMappingURL=loadArray.d.ts.map

package/script/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,QAiBnC;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,GACjC,aAAa,CA8Df;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+BxB"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA4ExB"}

package/script/methods/loadArray.js

@@ -7,16 +7,19 @@ exports.default = loadArray;
 exports.prepareArray = prepareArray;
 exports.executePreparedArray = executePreparedArray;
 const node_api_1 = require("@duckdb/node-api");
+const prepareGeometry_js_1 = __importDefault(require("../helpers/prepareGeometry.js"));
+const prepareJSON_js_1 = __importDefault(require("../helpers/prepareJSON.js"));
+const geometryFromJSON_js_1 = __importDefault(require("../helpers/geometryFromJSON.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const parseTypes_js_1 = __importDefault(require("../helpers/parseTypes.js"));
 const parseDuckDBType_js_1 = __importDefault(require("../helpers/parseDuckDBType.js"));
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
-function loadArray(simpleTable, rows) {
+function loadArray(simpleTable, rows, options = {}) {
     // This validation doesn't need the database, so it stays at call time.
     if (rows.length === 0) {
         throw new Error("The array is empty. loadArray needs at least one object to infer the column types.");
     }
-    const prepared = prepareArray(rows);
+    const prepared = prepareArray(rows, options.columnTypes, true);
     (0, queueOp_js_1.default)(simpleTable, {
         kind: "barrier",
         method: "loadArray()",
@@ -24,14 +27,55 @@ function loadArray(simpleTable, rows) {
         execute: () => executePreparedArray(simpleTable, prepared),
     });
 }
-function prepareArray(rows) {
+function prepareArray(rows, columnTypes = {}, requireExplicitNestedTypes = false) {
     const keys = Object.keys(rows[0]);
+    const overrides = new Map();
+    for (const [key, type] of Object.entries(columnTypes)) {
+        if (!keys.includes(key)) {
+            throw new Error(`Unknown column ${JSON.stringify(key)} in loadArray columnTypes.`);
+        }
+        if (type.toUpperCase() === "JSON") {
+            overrides.set(key, "JSON");
+            continue;
+        }
+        if (/^geometry/i.test(type)) {
+            if (!/^geometry\('EPSG:4326'\)$/i.test(type)) {
+                throw new Error(`Unsupported type ${JSON.stringify(type)} for column ${JSON.stringify(key)}. GeoJSON ingestion requires GEOMETRY('EPSG:4326'); reproject input to WGS84 first.`);
+            }
+            overrides.set(key, "GEOMETRY('EPSG:4326')");
+            continue;
+        }
+        try {
+            const normalized = (0, parseTypes_js_1.default)(type);
+            (0, parseDuckDBType_js_1.default)(normalized);
+            overrides.set(key, normalized);
+        }
+        catch {
+            throw new Error(`Unsupported type ${JSON.stringify(type)} for column ${JSON.stringify(key)}.`);
+        }
+    }
     const firstNonNullValue = keys.map((key) => rows.find((obj) => obj[key] !== null && obj[key] !== undefined)?.[key]);
     const types = [];
     const columnsData = keys.map(() => new Array(rows.length));
     for (let i = 0; i < keys.length; i++) {
         const key = keys[i];
         const columnData = columnsData[i];
+        const override = overrides.get(key);
+        if (override !== undefined) {
+            types[i] = override;
+            for (let j = 0; j < rows.length; j++) {
+                columnData[j] = prepareTypedValue(rows[j][key], override, key, j + 1);
+            }
+            continue;
+        }
+        if (requireExplicitNestedTypes) {
+            for (let j = 0; j < rows.length; j++) {
+                const cell = rows[j][key];
+                if (cell !== null && typeof cell === "object" && !(cell instanceof Date)) {
+                    throw new Error(`Column ${JSON.stringify(key)}, row ${j + 1}: array and object cells require an explicit columnTypes entry. Use JSON for nested data, FLOAT[n] for fixed-size vectors, or GEOMETRY('EPSG:4326') for GeoJSON geometries.`);
+                }
+            }
+        }
         const value = firstNonNullValue[i];
         const type = typeof value;
         if (type === "symbol" || type === "function") {
@@ -88,19 +132,139 @@ async function executePreparedArray(simpleTable, prepared) {
         simpleTable.connection = simpleTable.sdb.connection;
     }
     const { keys, types, columnsData, rowCount } = prepared;
-    await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${types[i]}`).join(", ")})`);
-    const appender = await simpleTable.connection
-        .createAppender(simpleTable.name);
-    const duckDBTypes = types.map((d) => (0, parseDuckDBType_js_1.default)(d));
-    // The maximum capacity of a DuckDB data chunk is 2048 rows.
-    const chunkSize = 2000;
-    for (let start = 0; start < rowCount; start += chunkSize) {
-        const end = Math.min(start + chunkSize, rowCount);
-        const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
-        for (let i = 0; i < keys.length; i++) {
-            dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-        }
-        appender.appendDataChunk(dataChunk);
+    const hasGeometry = types.includes("GEOMETRY('EPSG:4326')");
+    const needsConversion = hasGeometry || types.includes("JSON");
+    const staged = needsConversion
+        ? `__sda_array_${crypto.randomUUID().replaceAll("-", "")}`
+        : simpleTable.name;
+    const storageTypes = types.map((type) => type.startsWith("GEOMETRY") || type === "JSON" ? "VARCHAR" : type);
+    if (hasGeometry && !simpleTable.sdb.spatialLoaded) {
+        await simpleTable.sdb.customQuery("INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;");
+        simpleTable.sdb.spatialLoaded = true;
+    }
+    try {
+        await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${(0, quoteIdentifier_js_1.default)(staged)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${storageTypes[i]}`).join(", ")})`);
+        const appender = await simpleTable.connection
+            .createAppender(staged);
+        try {
+            const duckDBTypes = storageTypes.map((d) => (0, parseDuckDBType_js_1.default)(d));
+            // The maximum capacity of a DuckDB data chunk is 2048 rows.
+            const chunkSize = 2000;
+            for (let start = 0; start < rowCount; start += chunkSize) {
+                const end = Math.min(start + chunkSize, rowCount);
+                const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
+                for (let i = 0; i < keys.length; i++) {
+                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
+                }
+                appender.appendDataChunk(dataChunk);
+            }
+            appender.flushSync();
+        }
+        finally {
+            appender.closeSync();
+        }
+        if (!needsConversion)
+            return;
+        // CTAS is atomic: conversion must finish before the destination is replaced.
+        await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} AS SELECT ${keys.map((key, i) => types[i].startsWith("GEOMETRY")
+            ? `${(0, geometryFromJSON_js_1.default)((0, quoteIdentifier_js_1.default)(key))} AS ${(0, quoteIdentifier_js_1.default)(key)}`
+            : types[i] === "JSON"
+                ? `${(0, quoteIdentifier_js_1.default)(key)}::JSON AS ${(0, quoteIdentifier_js_1.default)(key)}`
+                : (0, quoteIdentifier_js_1.default)(key)).join(", ")} FROM ${(0, quoteIdentifier_js_1.default)(staged)}`);
+    }
+    finally {
+        if (needsConversion) {
+            await simpleTable.sdb.customQuery(`DROP TABLE IF EXISTS ${(0, quoteIdentifier_js_1.default)(staged)}`);
+        }
+    }
+}
+/** Validates and snapshots a value for an explicitly declared destination type. */
+function prepareTypedValue(value, type, column, row) {
+    if (type === "JSON")
+        return (0, prepareJSON_js_1.default)(value, column, row);
+    if (type === "GEOMETRY('EPSG:4326')") {
+        return (0, prepareGeometry_js_1.default)(value, column, row);
+    }
+    if (value === null || value === undefined || Number.isNaN(value))
+        return null;
+    let compatible = false;
+    if (type === "VARCHAR" && typeof value === "string")
+        return value;
+    if (type === "BOOLEAN" && typeof value === "boolean")
+        return value;
+    if (type === "DOUBLE") {
+        if (typeof value === "number")
+            return value;
+        compatible = typeof value === "bigint";
+        if (typeof value === "bigint") {
+            const number = Number(value);
+            if (Number.isFinite(number) && BigInt(number) === value)
+                return number;
+        }
+    }
+    else if (type === "INTEGER") {
+        compatible = typeof value === "number" || typeof value === "bigint";
+        if (typeof value === "number") {
+            if (Number.isInteger(value) && value >= -2147483648 && value <= 2147483647) {
+                return value;
+            }
+        }
+        else if (typeof value === "bigint" && value >= -2147483648n && value <= 2147483647n) {
+            return Number(value);
+        }
+    }
+    else if (type === "BIGINT") {
+        compatible = typeof value === "number" || typeof value === "bigint";
+        if (typeof value === "bigint") {
+            if (value >= -9223372036854775808n && value <= 9223372036854775807n) {
+                return value;
+            }
+        }
+        else if (typeof value === "number" && Number.isInteger(value) &&
+            value >= -(2 ** 63) && value < 2 ** 63) {
+            // The upper bound is exclusive: 2 ** 63 - 1 rounds up as a JS number.
+            return BigInt(value);
+        }
+    }
+    else if (["DATE", "TIME", "TIMESTAMP", "TIMESTAMP WITH TIME ZONE"].includes(type)) {
+        if (type === "TIME" && value instanceof node_api_1.DuckDBTimeValue) {
+            if (value.micros >= 0n && value.micros <= 86400000000n) {
+                return new node_api_1.DuckDBTimeValue(value.micros);
+            }
+            compatible = true;
+        }
+        else if (value instanceof Date) {
+            const millis = value.getTime();
+            compatible = Number.isFinite(millis);
+            if (compatible) {
+                if (type === "DATE" && millis % 86400000 === 0) {
+                    return new node_api_1.DuckDBDateValue(millis / 86400000);
+                }
+                const micros = BigInt(millis) * 1000n;
+                if (type === "TIMESTAMP")
+                    return new node_api_1.DuckDBTimestampValue(micros);
+                if (type === "TIMESTAMP WITH TIME ZONE") {
+                    return new node_api_1.DuckDBTimestampTZValue(micros);
+                }
+            }
+        }
+    }
+    else if (type.startsWith("FLOAT[") && Array.isArray(value)) {
+        compatible = true;
+        const size = Number(type.slice(6, -1));
+        if (value.length === size &&
+            value.every((item) => item === null || item === undefined ||
+                (typeof item === "number" &&
+                    (Number.isNaN(item) || Math.fround(item) === item)))) {
+            return (0, node_api_1.arrayValue)(value.map((item) => item === undefined || Number.isNaN(item)
+                ? null
+                : item));
+        }
     }
-    appender.flushSync();
+    const displayed = typeof value === "string"
+        ? JSON.stringify(value)
+        : String(value);
+    throw new Error(`Column ${JSON.stringify(column)}, row ${row}: value ${displayed} ${compatible
+        ? `cannot be stored as ${type} without losing information`
+        : `is incompatible with ${type}`}. Select a type compatible with this value.`);
 }

package/script/methods/loadData.js

@@ -54,6 +54,9 @@ function loadDataQuery(table, files, options = {}) {
     return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table)} AS ${loadDataSelect(files, options)};`;
 }
 function loadDataSelect(files, options = {}) {
+    if (Object.values(options.columnTypes ?? {}).some((type) => /^\s*geometry\b/i.test(type))) {
+        throw new Error("Geometry column types are not supported by loadData(). Use loadGeoData() instead.");
+    }
     const fileExtension = (0, getExtension_js_1.default)(files[0]);
     if (options.fileType !== undefined &&
         !["csv", "dsv", "json", "parquet", "excel"].includes(options.fileType)) {

package/script/methods/loadYahooFinanceData.d.ts

@@ -0,0 +1,10 @@
+import type SimpleTable from "../class/SimpleTable.js";
+declare const INTERVAL_SECONDS: {
+    readonly "1d": number;
+    readonly "1h": number;
+    readonly "1m": 60;
+};
+type YahooInterval = keyof typeof INTERVAL_SECONDS;
+export default function loadYahooFinanceData(table: SimpleTable, symbol: string, startDate: Date, endDate: Date, interval: YahooInterval): void;
+export {};
+//# sourceMappingURL=loadYahooFinanceData.d.ts.map

package/script/methods/loadYahooFinanceData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadYahooFinanceData.d.ts","sourceRoot":"","sources":["../../src/methods/loadYahooFinanceData.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,QAAA,MAAM,gBAAgB;;;;CAIZ,CAAC;AAeX,KAAK,aAAa,GAAG,MAAM,OAAO,gBAAgB,CAAC;AA6BnD,MAAM,CAAC,OAAO,UAAU,oBAAoB,CAC1C,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,IAAI,EACf,OAAO,EAAE,IAAI,EACb,QAAQ,EAAE,aAAa,GACtB,IAAI,CAeN"}

package/script/methods/loadYahooFinanceData.js

@@ -0,0 +1,139 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadYahooFinanceData;
+const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const loadArray_js_1 = require("./loadArray.js");
+const INTERVAL_SECONDS = {
+    "1d": 24 * 60 * 60,
+    "1h": 60 * 60,
+    "1m": 60,
+};
+const COLUMN_TYPES = {
+    datetime: "TIMESTAMP",
+    open: "DOUBLE",
+    high: "DOUBLE",
+    low: "DOUBLE",
+    close: "DOUBLE",
+    adjustedClose: "DOUBLE",
+    volume: "DOUBLE",
+};
+const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
+function loadYahooFinanceData(table, symbol, startDate, endDate, interval) {
+    assertArguments(symbol, startDate, endDate, interval);
+    const request = {
+        symbol,
+        startDate: new Date(startDate),
+        endDate: new Date(endDate),
+        interval,
+    };
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "loadYahooFinanceData()",
+        parameters: request,
+        execute: () => executeLoadYahooFinanceData(table, request),
+    });
+}
+async function executeLoadYahooFinanceData(table, request) {
+    try {
+        const rows = await getYahooFinanceData(request);
+        await (0, loadArray_js_1.executePreparedArray)(table, (0, loadArray_js_1.prepareArray)(rows, COLUMN_TYPES));
+    }
+    catch (error) {
+        if (error instanceof SDAError_js_1.default) {
+            throw error;
+        }
+        throw new SDAError_js_1.default({
+            method: "loadYahooFinanceData()",
+            parameters: request,
+            query: "",
+            cause: error,
+        });
+    }
+}
+async function getYahooFinanceData(request) {
+    const { symbol, startDate, endDate, interval } = request;
+    const startTime = startDate.getTime();
+    const exclusiveEndTime = getExclusiveEndTime(endDate, interval);
+    const url = new URL(`https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`);
+    url.search = new URLSearchParams({
+        includeAdjustedClose: "true",
+        interval,
+        period1: String(Math.floor(startTime / 1000)),
+        period2: String(Math.ceil(exclusiveEndTime / 1000)),
+    }).toString();
+    const response = await fetch(url, {
+        headers: { "User-Agent": USER_AGENT },
+    });
+    if (!response.ok) {
+        const text = (await response.text()).slice(0, 500).trim();
+        throw new Error(`Failed to fetch Yahoo Finance data: ${response.status} ${response.statusText}${text.length > 0 ? `. ${text}` : ""}. Yahoo may have changed, rate-limited, or disabled this undocumented endpoint.`);
+    }
+    const data = await response.json();
+    if (data.chart?.error) {
+        throw new Error(data.chart.error.description ??
+            "Yahoo Finance returned an unknown error.");
+    }
+    const result = data.chart?.result?.[0];
+    const timestamps = result?.timestamp;
+    if (!result || !timestamps?.length) {
+        throw new Error("No Yahoo Finance data found.");
+    }
+    const quote = result.indicators?.quote?.[0];
+    const adjustedClose = result.indicators?.adjclose?.[0]?.adjclose;
+    const rows = [];
+    for (let index = 0; index < timestamps.length; index++) {
+        const timestamp = timestamps[index] * 1000;
+        if (Number.isFinite(timestamp) && timestamp >= startTime &&
+            timestamp < exclusiveEndTime) {
+            rows.push({
+                datetime: new Date(timestamp),
+                open: getFiniteValue(quote?.open, index),
+                high: getFiniteValue(quote?.high, index),
+                low: getFiniteValue(quote?.low, index),
+                close: getFiniteValue(quote?.close, index),
+                adjustedClose: getFiniteValue(adjustedClose, index),
+                volume: getFiniteValue(quote?.volume, index),
+            });
+        }
+    }
+    if (rows.length === 0) {
+        throw new Error(`No Yahoo Finance data found for ${symbol}.`);
+    }
+    return rows;
+}
+function getFiniteValue(values, index) {
+    const value = values?.[index];
+    return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function getExclusiveEndTime(endDate, interval) {
+    const end = new Date(endDate);
+    if (interval === "1d") {
+        end.setUTCHours(0, 0, 0, 0);
+    }
+    else if (interval === "1h") {
+        end.setUTCMinutes(0, 0, 0);
+    }
+    else {
+        end.setUTCSeconds(0, 0);
+    }
+    return end.getTime() + INTERVAL_SECONDS[interval] * 1000;
+}
+function assertArguments(symbol, startDate, endDate, interval) {
+    if (typeof symbol !== "string" || symbol.trim().length === 0) {
+        throw new TypeError("loadYahooFinanceData() symbol must be a non-empty string.");
+    }
+    if (!(startDate instanceof Date) || !(endDate instanceof Date) ||
+        !Number.isFinite(startDate.getTime()) || !Number.isFinite(endDate.getTime())) {
+        throw new RangeError("loadYahooFinanceData() startDate and endDate must be valid dates.");
+    }
+    if (endDate.getTime() < startDate.getTime()) {
+        throw new RangeError("loadYahooFinanceData() endDate must be equal to or later than startDate.");
+    }
+    if (interval !== "1d" && interval !== "1h" && interval !== "1m") {
+        throw new Error('loadYahooFinanceData() interval must be "1d", "1h", or "1m".');
+    }
+}

package/script/methods/log.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function log(simpleTable: SimpleTable, options?: "all" | number | {
+    count?: number | "all";
+    types?: boolean;
+    conditions?: string;
+}): Promise<void>;
+//# sourceMappingURL=log.d.ts.map

package/script/methods/log.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../../src/methods/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,GAAG,CAC/B,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,KAAK,GAAG,MAAM,GAAG;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,iBAyEP"}