@nshiab/simple-data-analysis-core 1.0.1 → 2.0.1

package/script/methods/linearRegressions.js

@@ -4,46 +4,100 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = linearRegressions;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const getCombinations_js_1 = __importDefault(require("../helpers/getCombinations.js"));
-const keepNumericalColumns_js_1 = __importDefault(require("../helpers/keepNumericalColumns.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
-const linearRegressionQuery_js_1 = __importDefault(require("./linearRegressionQuery.js"));
-async function linearRegressions(SimpleTable, options = {}) {
-    const outputTable = typeof options.outputTable === "string"
-        ? options.outputTable
-        : SimpleTable.name;
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const resolveOutputTable_js_1 = __importDefault(require("../helpers/resolveOutputTable.js"));
+const stringToArray_js_1 = __importDefault(require("../helpers/stringToArray.js"));
+const validateNumericalAnalysis_js_1 = __importDefault(require("../helpers/validateNumericalAnalysis.js"));
+function linearRegressions(simpleTable, options = {}) {
+    options = structuredClone(options);
+    options.outputTable = (0, resolveOutputTable_js_1.default)(simpleTable, options.outputTable);
+    if (typeof options.outputTable === "string") {
+        // The output table instance is created at call time so it can be
+        // returned synchronously and chained on right away.
+        const outputTable = simpleTable.sdb.newTable(options.outputTable);
+        (0, queueOp_js_1.default)(outputTable, {
+            kind: "barrier",
+            method: "linearRegressions()",
+            parameters: { options },
+            execute: async () => {
+                const permutations = getLinearRegressionsPermutations(await simpleTable.getTypes(), options, simpleTable.name);
+                await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable.name)} AS ${linearRegressionsSelect(`${(0, quoteIdentifier_js_1.default)(simpleTable.name)}`, permutations, options)}`, (0, mergeOptions_js_1.default)(simpleTable, {
+                    table: outputTable.name,
+                    method: "linearRegressions()",
+                    parameters: {
+                        options,
+                        "permutations (computed)": permutations,
+                    },
+                    values: linearRegressionsValues(permutations),
+                }));
+            },
+        });
+        return outputTable;
+    }
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "linearRegressions()",
+        parameters: { options },
+        needsSchema: true,
+        values: (types) => linearRegressionsValues(getLinearRegressionsPermutations(types, options, simpleTable.name)),
+        buildSelect: (input, types) => linearRegressionsSelect(input, getLinearRegressionsPermutations(types, options, simpleTable.name), options),
+    });
+    return simpleTable;
+}
+function linearRegressionsValues(permutations) {
+    return permutations.flatMap(([x, y]) => [x, y]);
+}
+function getLinearRegressionsPermutations(types, options, tableName) {
+    const columns = (0, validateNumericalAnalysis_js_1.default)(types, options, "linearRegressions()", tableName);
     const permutations = [];
-    if (!options.x && !options.y) {
-        const types = await SimpleTable.getTypes();
-        const columns = (0, keepNumericalColumns_js_1.default)(types);
+    if (options.x === undefined) {
         const combinations = (0, getCombinations_js_1.default)(columns, 2);
         for (const c of combinations) {
             permutations.push(c);
             permutations.push([c[1], c[0]]);
         }
     }
-    else if (options.x && !options.y) {
-        const types = await SimpleTable.getTypes();
-        const columns = (0, keepNumericalColumns_js_1.default)(types);
+    else if (options.y === undefined) {
         for (const col of columns) {
             if (col !== options.x) {
                 permutations.push([options.x, col]);
             }
         }
     }
-    else if (options.x && options.y) {
-        permutations.push([options.x, options.y]);
-    }
     else {
-        throw new Error("No combinations of x and y");
+        permutations.push([options.x, options.y]);
     }
-    await (0, queryDB_js_1.default)(SimpleTable, (0, linearRegressionQuery_js_1.default)(SimpleTable.name, outputTable, permutations, options), (0, mergeOptions_js_1.default)(SimpleTable, {
-        table: outputTable,
-        method: "linearRegressions()",
-        parameters: {
-            options,
-            "permutations (computed)": permutations,
-        },
-    }));
+    return permutations;
+}
+function linearRegressionsSelect(input, permutations, options) {
+    const by = options.by ? (0, stringToArray_js_1.default)(options.by) : [];
+    const groupBy = by.length === 0
+        ? ""
+        : ` GROUP BY ${by.map((d) => `${(0, quoteIdentifier_js_1.default)(d)}`).join(",")}`;
+    const catSelect = by.length > 0
+        ? `${by.map((d) => `${(0, quoteIdentifier_js_1.default)(d)}`).join(",")}, `
+        : "";
+    // One UNION ALL branch per permutation (each with a distinct pair of
+    // literals, so deduplication would never remove anything). DuckDB runs the
+    // branches as concurrent pipelines, which parallelizes the aggregates
+    // better than computing them all in one scan.
+    const branches = permutations.map((perm) => {
+        const expressions = [
+            ["slope", "REGR_SLOPE"],
+            ["yIntercept", "REGR_INTERCEPT"],
+            ["r2", "REGR_R2"],
+        ].map(([alias, fn]) => {
+            const expression = typeof options.decimals === "number"
+                ? `ROUND(${fn}(${(0, quoteIdentifier_js_1.default)(perm[1])}, ${(0, quoteIdentifier_js_1.default)(perm[0])}), ${options.decimals})`
+                : `${fn}(${(0, quoteIdentifier_js_1.default)(perm[1])}, ${(0, quoteIdentifier_js_1.default)(perm[0])})`;
+            return `${expression} AS ${(0, quoteIdentifier_js_1.default)(alias)}`;
+        });
+        return `SELECT ${catSelect}? AS ${(0, quoteIdentifier_js_1.default)("x")}, ? AS ${(0, quoteIdentifier_js_1.default)("y")}, ${expressions.join(", ")}
+        FROM ${input}${groupBy}`;
+    });
+    return branches.join("\nUNION ALL\n");
 }

package/script/methods/linesToPolygons.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function linesToPolygons(simpleTable: SimpleTable, column?: string): void;
+//# sourceMappingURL=linesToPolygons.d.ts.map

package/script/methods/linesToPolygons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"linesToPolygons.d.ts","sourceRoot":"","sources":["../../src/methods/linesToPolygons.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,WAAW,EAAE,WAAW,EACxB,MAAM,CAAC,EAAE,MAAM,QAgBhB"}

package/script/methods/linesToPolygons.js

@@ -0,0 +1,23 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = linesToPolygons;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const findGeoColumnFromSchema_js_1 = __importDefault(require("../helpers/findGeoColumnFromSchema.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+function linesToPolygons(simpleTable, column) {
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "fusable",
+        method: "linesToPolygons()",
+        parameters: { column },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            const col = column ??
+                (0, findGeoColumnFromSchema_js_1.default)(types, "linesToPolygons()", simpleTable.name);
+            return `SELECT * EXCLUDE(${(0, quoteIdentifier_js_1.default)(col)}), ST_MakePolygon(${(0, quoteIdentifier_js_1.default)(col)}) as ${(0, quoteIdentifier_js_1.default)(col)} FROM ${input}`;
+        },
+    });
+}

package/script/methods/loadArray.d.ts

@@ -1,5 +1,17 @@
+import { type DuckDBValue } from "@duckdb/node-api";
 import type SimpleTable from "../class/SimpleTable.js";
-export default function loadArray(simpleTable: SimpleTable, arrayOfObjects: {
+export default function loadArray(simpleTable: SimpleTable, rows: {
     [key: string]: unknown;
-}[]): Promise<void>;
+}[]): void;
+type PreparedArray = {
+    keys: string[];
+    types: string[];
+    columnsData: DuckDBValue[][];
+    rowCount: number;
+};
+export declare function prepareArray(rows: {
+    [key: string]: unknown;
+}[]): PreparedArray;
+export declare function executePreparedArray(simpleTable: SimpleTable, prepared: PreparedArray): Promise<void>;
+export {};
 //# sourceMappingURL=loadArray.d.ts.map

package/script/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,SAAS,CACrC,WAAW,EAAE,WAAW,EACxB,cAAc,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,iBAkG7C"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,QAiBnC;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,GACjC,aAAa,CA8Df;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+BxB"}

package/script/methods/loadArray.js

@@ -4,52 +4,66 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = loadArray;
+exports.prepareArray = prepareArray;
+exports.executePreparedArray = executePreparedArray;
 const node_api_1 = require("@duckdb/node-api");
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const parseTypes_js_1 = __importDefault(require("../helpers/parseTypes.js"));
 const parseDuckDBType_js_1 = __importDefault(require("../helpers/parseDuckDBType.js"));
-async function loadArray(simpleTable, arrayOfObjects) {
-    if (simpleTable.connection === undefined) {
-        await simpleTable.sdb.start();
-        simpleTable.connection = simpleTable.sdb.connection;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+function loadArray(simpleTable, rows) {
+    // This validation doesn't need the database, so it stays at call time.
+    if (rows.length === 0) {
+        throw new Error("The array is empty. loadArray needs at least one object to infer the column types.");
     }
-    const keys = Object.keys(arrayOfObjects[0]);
-    const firstNonNullValue = keys.map((key) => arrayOfObjects.find((obj) => obj[key] !== null && obj[key] !== undefined)?.[key]);
+    const prepared = prepareArray(rows);
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "barrier",
+        method: "loadArray()",
+        parameters: { rows: `${rows.length} rows` },
+        execute: () => executePreparedArray(simpleTable, prepared),
+    });
+}
+function prepareArray(rows) {
+    const keys = Object.keys(rows[0]);
+    const firstNonNullValue = keys.map((key) => rows.find((obj) => obj[key] !== null && obj[key] !== undefined)?.[key]);
     const types = [];
-    const dataForChunk = arrayOfObjects.map(() => []);
+    const columnsData = keys.map(() => new Array(rows.length));
     for (let i = 0; i < keys.length; i++) {
         const key = keys[i];
+        const columnData = columnsData[i];
         const value = firstNonNullValue[i];
         const type = typeof value;
-        if (type === "symbol" ||
-            type === "function") {
+        if (type === "symbol" || type === "function") {
             throw new Error(`Type ${type} not supported for ${key}. Value: ${value}`);
         }
         else if (type === "undefined") {
-            // If all values in the column are null or undefined, default to VARCHAR type.
             types[i] = "VARCHAR";
-            for (let j = 0; j < arrayOfObjects.length; j++) {
-                dataForChunk[j][i] = null;
+            for (let j = 0; j < rows.length; j++) {
+                columnData[j] = null;
             }
         }
         else if (type === "object") {
             if (value instanceof Date) {
                 types[i] = "TIMESTAMP";
-                for (let j = 0; j < arrayOfObjects.length; j++) {
-                    const d = arrayOfObjects[j][key];
+                for (let j = 0; j < rows.length; j++) {
+                    const d = rows[j][key];
                     if (d === null || d === undefined || Number.isNaN(d)) {
-                        dataForChunk[j][i] = null;
+                        columnData[j] = null;
                     }
                     else {
                         const date = d;
-                        dataForChunk[j][i] = new node_api_1.DuckDBTimestampValue(BigInt(date.getTime() * 1000));
+                        columnData[j] = new node_api_1.DuckDBTimestampValue(BigInt(date.getTime() * 1000));
                     }
                 }
             }
             else if (Array.isArray(value)) {
                 types[i] = `FLOAT[${value.length}]`;
-                for (let j = 0; j < arrayOfObjects.length; j++) {
-                    const d = arrayOfObjects[j][key];
-                    dataForChunk[j][i] = (0, node_api_1.arrayValue)(d);
+                for (let j = 0; j < rows.length; j++) {
+                    const d = rows[j][key];
+                    columnData[j] = d === null || d === undefined
+                        ? null
+                        : (0, node_api_1.arrayValue)([...d]);
                 }
             }
             else {
@@ -58,26 +72,34 @@ async function loadArray(simpleTable, arrayOfObjects) {
         }
         else {
             types[i] = (0, parseTypes_js_1.default)(type);
-            for (let j = 0; j < arrayOfObjects.length; j++) {
-                const d = arrayOfObjects[j][key];
-                if (d === null || d === undefined || Number.isNaN(d)) {
-                    dataForChunk[j][i] = null;
-                }
-                else {
-                    dataForChunk[j][i] = d;
-                }
+            for (let j = 0; j < rows.length; j++) {
+                const d = rows[j][key];
+                columnData[j] = d === null || d === undefined || Number.isNaN(d)
+                    ? null
+                    : d;
             }
         }
     }
-    await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE "${simpleTable.name}"(${keys.map((key, i) => `"${key}" ${types[i]}`).join(", ")})`);
+    return { keys, types, columnsData, rowCount: rows.length };
+}
+async function executePreparedArray(simpleTable, prepared) {
+    if (simpleTable.connection === undefined) {
+        await simpleTable.sdb.start();
+        simpleTable.connection = simpleTable.sdb.connection;
+    }
+    const { keys, types, columnsData, rowCount } = prepared;
+    await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${types[i]}`).join(", ")})`);
     const appender = await simpleTable.connection
         .createAppender(simpleTable.name);
+    const duckDBTypes = types.map((d) => (0, parseDuckDBType_js_1.default)(d));
     // The maximum capacity of a DuckDB data chunk is 2048 rows.
     const chunkSize = 2000;
-    for (let i = 0; i < dataForChunk.length; i += chunkSize) {
-        const chunk = dataForChunk.slice(i, i + chunkSize);
-        const dataChunk = node_api_1.DuckDBDataChunk.create(types.map((d) => (0, parseDuckDBType_js_1.default)(d)), chunk.length);
-        dataChunk.setRows(chunk);
+    for (let start = 0; start < rowCount; start += chunkSize) {
+        const end = Math.min(start + chunkSize, rowCount);
+        const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
+        for (let i = 0; i < keys.length; i++) {
+            dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
+        }
         appender.appendDataChunk(dataChunk);
     }
     appender.flushSync();

package/script/methods/loadDB.d.ts

@@ -0,0 +1,3 @@
+import type SimpleDB from "../class/SimpleDB.js";
+export default function loadDB(sdb: SimpleDB, file: string): Promise<void>;
+//# sourceMappingURL=loadDB.d.ts.map

package/script/methods/loadDB.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadDB.d.ts","sourceRoot":"","sources":["../../src/methods/loadDB.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAYjD,wBAA8B,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,iBA0E/D"}

package/script/methods/loadDB.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadDB;
+const queryDbFile_js_1 = __importDefault(require("../helpers/queryDbFile.js"));
+const node_fs_1 = require("node:fs");
+const cleanPath_js_1 = __importDefault(require("../helpers/cleanPath.js"));
+const getDbFileType_js_1 = __importDefault(require("../helpers/getDbFileType.js"));
+const getCurrentDatabase_js_1 = __importDefault(require("../helpers/getCurrentDatabase.js"));
+const getDbIndexes_js_1 = __importDefault(require("../helpers/getDbIndexes.js"));
+const parseValue_js_1 = __importDefault(require("../helpers/parseValue.js"));
+const prepareDbExtensions_js_1 = __importDefault(require("../helpers/prepareDbExtensions.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const readDbMetadata_js_1 = __importDefault(require("../helpers/readDbMetadata.js"));
+const setDbProps_js_1 = __importDefault(require("../helpers/setDbProps.js"));
+const writeDbMetadata_js_1 = __importDefault(require("../helpers/writeDbMetadata.js"));
+async function loadDB(sdb, file) {
+    const type = (0, getDbFileType_js_1.default)(file);
+    if (!(0, node_fs_1.existsSync)(file))
+        throw new Error(`The file ${file} does not exist.`);
+    if (sdb.readOnly) {
+        throw new Error("loadDB() cannot import into a read-only database.");
+    }
+    const destination = await (0, getCurrentDatabase_js_1.default)(sdb);
+    const target = (0, quoteIdentifier_js_1.default)(destination);
+    const name = `__sda_import_${crypto.randomUUID().replaceAll("-", "")}`;
+    const source = (0, quoteIdentifier_js_1.default)(name);
+    if (type === "sqlite")
+        await (0, queryDbFile_js_1.default)(sdb, "INSTALL sqlite; LOAD sqlite;");
+    await (0, queryDbFile_js_1.default)(sdb, `ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${source} (TYPE ${type}, READ_ONLY);`);
+    let inTransaction = false;
+    try {
+        const indexes = type === "duckdb" ? await (0, readDbMetadata_js_1.default)(sdb, name) : {};
+        await (0, readDbMetadata_js_1.default)(sdb, destination);
+        await (0, prepareDbExtensions_js_1.default)(sdb, name, indexes);
+        const conflicts = await (0, queryDbFile_js_1.default)(sdb, `SELECT incoming.table_schema, incoming.table_name
+       FROM information_schema.tables incoming
+       JOIN information_schema.tables existing
+         ON lower(incoming.table_schema) = lower(existing.table_schema)
+         AND lower(incoming.table_name) = lower(existing.table_name)
+       WHERE incoming.table_catalog = ${(0, parseValue_js_1.default)(name)}
+         AND existing.table_catalog = ${(0, parseValue_js_1.default)(destination)}
+         AND incoming.table_schema <> '__sda';`, { returnData: true });
+        // Also protect registered table handles that have no physical table yet.
+        const incoming = await (0, queryDbFile_js_1.default)(sdb, `SELECT table_name FROM information_schema.tables
+       WHERE table_catalog = ${(0, parseValue_js_1.default)(name)} AND table_schema = 'main';`, { returnData: true });
+        const registered = new Set(sdb.getTables().map((table) => table.name.toLowerCase()));
+        const names = new Set([
+            ...conflicts.map((row) => `${row.table_schema}.${row.table_name}`),
+            ...incoming.filter((row) => registered.has(String(row.table_name).toLowerCase()))
+                .map((row) => `main.${row.table_name}`),
+        ]);
+        if (names.size > 0) {
+            throw new Error(`loadDB() cannot import existing tables: ${[...names].join(", ")}.`);
+        }
+        const mergedIndexes = { ...(0, getDbIndexes_js_1.default)(sdb), ...indexes };
+        await (0, queryDbFile_js_1.default)(sdb, "BEGIN TRANSACTION;");
+        inTransaction = true;
+        await (0, queryDbFile_js_1.default)(sdb, `DROP TABLE IF EXISTS ${target}.__sda.metadata;
+       COPY FROM DATABASE ${source} TO ${target};`);
+        await (0, writeDbMetadata_js_1.default)(sdb, destination, mergedIndexes);
+        await (0, queryDbFile_js_1.default)(sdb, "COMMIT;");
+        inTransaction = false;
+        await (0, setDbProps_js_1.default)(sdb, indexes);
+    }
+    catch (error) {
+        if (inTransaction)
+            await (0, queryDbFile_js_1.default)(sdb, "ROLLBACK;");
+        throw error;
+    }
+    finally {
+        await (0, queryDbFile_js_1.default)(sdb, `DETACH ${source};`);
+    }
+}

package/script/methods/loadData.d.ts

@@ -0,0 +1,52 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function loadData(simpleTable: SimpleTable, files: string | string[], options?: {
+    fileType?: "csv" | "dsv" | "json" | "parquet" | "excel";
+    autoDetect?: boolean;
+    conditions?: string;
+    limit?: number;
+    includeFilename?: boolean;
+    unifyColumns?: boolean;
+    columnTypes?: {
+        [key: string]: string;
+    };
+    columns?: string[];
+    header?: boolean;
+    allText?: boolean;
+    delim?: string;
+    skip?: number;
+    nullPadding?: boolean;
+    ignoreErrors?: boolean;
+    compression?: "none" | "gzip" | "zstd";
+    encoding?: string;
+    strict?: boolean;
+    jsonFormat?: "unstructured" | "newlineDelimited" | "array";
+    records?: boolean;
+    sheet?: string;
+}): void;
+type LoadDataOptions = {
+    fileType?: "csv" | "dsv" | "json" | "parquet" | "excel";
+    autoDetect?: boolean;
+    conditions?: string;
+    limit?: number;
+    includeFilename?: boolean;
+    unifyColumns?: boolean;
+    columnTypes?: {
+        [key: string]: string;
+    };
+    columns?: string[];
+    header?: boolean;
+    allText?: boolean;
+    delim?: string;
+    skip?: number;
+    nullPadding?: boolean;
+    ignoreErrors?: boolean;
+    compression?: "none" | "gzip" | "zstd";
+    encoding?: string;
+    strict?: boolean;
+    jsonFormat?: "unstructured" | "newlineDelimited" | "array";
+    records?: boolean;
+    sheet?: string;
+};
+export declare function loadDataQuery(table: string, files: string[], options?: LoadDataOptions): string;
+export {};
+//# sourceMappingURL=loadData.d.ts.map

package/script/methods/loadData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadData.d.ts","sourceRoot":"","sources":["../../src/methods/loadData.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,WAAW,EAAE,WAAW,EACxB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EACxB,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAExC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,UAAU,CAAC,EAAE,cAAc,GAAG,kBAAkB,GAAG,OAAO,CAAC;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,QAgBP;AAED,KAAK,eAAe,GAAG;IACrB,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACxC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,cAAc,GAAG,kBAAkB,GAAG,OAAO,CAAC;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAsCF,wBAAgB,aAAa,CAC3B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,eAAoB,UAK9B"}

package/script/methods/loadData.js

@@ -0,0 +1,155 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadData;
+exports.loadDataQuery = loadDataQuery;
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const parseValue_js_1 = __importDefault(require("../helpers/parseValue.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const getExtension_js_1 = __importDefault(require("../helpers/getExtension.js"));
+const stringToArray_js_1 = __importDefault(require("../helpers/stringToArray.js"));
+function loadData(simpleTable, files, options = {}) {
+    files = Array.isArray(files) ? [...files] : files;
+    options = structuredClone(options);
+    // Building the SELECT doesn't need the database, so invalid arguments
+    // (like the columns option with an Excel file) throw at call time.
+    const select = loadDataSelect((0, stringToArray_js_1.default)(files), options);
+    (0, queueOp_js_1.default)(simpleTable, {
+        kind: "source",
+        method: "loadData()",
+        parameters: { files, options },
+        rawSQL: options.conditions === undefined ? undefined : [options.conditions],
+        schema: knownSourceSchema((0, stringToArray_js_1.default)(files), options),
+        buildSelect: () => select,
+    });
+}
+function knownSourceSchema(files, options) {
+    if (options.columns === undefined || options.columns.length === 0) {
+        return undefined;
+    }
+    const extension = (0, getExtension_js_1.default)(files[0]);
+    const textSource = options.fileType === "csv" ||
+        options.fileType === "dsv" ||
+        options.fileType === "excel" ||
+        extension === "csv" ||
+        extension === "xlsx" ||
+        typeof options.delim === "string";
+    if (!textSource)
+        return undefined;
+    if (options.allText === true) {
+        return Object.fromEntries(options.columns.map((column) => [column, "VARCHAR"]));
+    }
+    if (options.columnTypes !== undefined &&
+        options.columns.every((column) => options.columnTypes?.[column] !== undefined)) {
+        return Object.fromEntries(options.columns.map((column) => [
+            column,
+            options.columnTypes[column].toUpperCase(),
+        ]));
+    }
+    return undefined;
+}
+function loadDataQuery(table, files, options = {}) {
+    return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table)} AS ${loadDataSelect(files, options)};`;
+}
+function loadDataSelect(files, options = {}) {
+    const fileExtension = (0, getExtension_js_1.default)(files[0]);
+    if (options.fileType !== undefined &&
+        !["csv", "dsv", "json", "parquet", "excel"].includes(options.fileType)) {
+        throw unsupportedFileTypeError(files[0], options.fileType, fileExtension);
+    }
+    const filesAsString = "[" +
+        files.map((file) => (0, parseValue_js_1.default)(file)).join(", ") +
+        "]";
+    // Column selection: throw for Excel, build SELECT list for others
+    const isExcel = options.fileType === "excel" || fileExtension === "xlsx";
+    if (options.columns && options.columns.length > 0 && isExcel) {
+        throw new Error("The 'columns' option is not supported for Excel files.");
+    }
+    const selectColumns = options.columns && options.columns.length > 0
+        ? options.columns.map((c) => `${(0, quoteIdentifier_js_1.default)(c)}`).join(", ")
+        : "*";
+    // General options, except for parquet
+    const autoDetect = typeof options.autoDetect === "boolean"
+        ? `, auto_detect=${String(options.autoDetect).toUpperCase()}`
+        : ", auto_detect=TRUE";
+    const columnTypes = options.columnTypes
+        ? `${Object.entries(options.columnTypes).map(([column, type]) => `${(0, parseValue_js_1.default)(column)}: ${(0, parseValue_js_1.default)(type)}`).join(", ")}`
+        : null;
+    const includeFilename = typeof options.includeFilename === "boolean"
+        ? `, filename=${String(options.includeFilename).toUpperCase()}`
+        : "";
+    const unifyColumns = typeof options.unifyColumns === "boolean"
+        ? `, union_by_name=${String(options.unifyColumns).toUpperCase()}`
+        : "";
+    const generalOptions = `${autoDetect}${includeFilename}${unifyColumns}`;
+    const conditions = options.conditions ? ` WHERE ${options.conditions}` : "";
+    const limit = typeof options.limit === "number"
+        ? ` LIMIT ${options.limit}`
+        : "";
+    if (options.fileType === "csv" ||
+        fileExtension === "csv" ||
+        options.fileType === "dsv" ||
+        typeof options.delim === "string") {
+        const header = typeof options.header === "boolean"
+            ? `, header=${String(options.header).toUpperCase()}`
+            : ", header=TRUE";
+        const allText = typeof options.allText === "boolean"
+            ? `, all_varchar=${String(options.allText).toUpperCase()}`
+            : "";
+        const delim = options.delim ? `, delim=${(0, parseValue_js_1.default)(options.delim)}` : "";
+        const skip = options.skip ? `, skip=${options.skip}` : "";
+        const ignoreErrors = options.ignoreErrors
+            ? `, ignore_errors	=${options.ignoreErrors}`
+            : "";
+        const nullPadding = options.nullPadding
+            ? `, null_padding=${options.nullPadding}`
+            : "";
+        const compression = options.compression
+            ? `, compression=${(0, parseValue_js_1.default)(options.compression)}`
+            : "";
+        const encoding = options.encoding
+            ? `, encoding=${(0, parseValue_js_1.default)(options.encoding)}`
+            : "";
+        const strict = options.strict === false ? `, strict_mode=FALSE` : "";
+        const types = columnTypes === null ? "" : `, types={${columnTypes}}`;
+        return `SELECT ${selectColumns} FROM read_csv_auto(${filesAsString}${generalOptions}${types}${header}${allText}${delim}${skip}${compression}${encoding}${strict}${nullPadding}${ignoreErrors})${conditions}${limit}`;
+    }
+    else if (options.fileType === "json" || fileExtension === "json") {
+        // DuckDB expects "newline_delimited" (snake_case), unlike the other two
+        // format values, which already match the public camelCase option.
+        const jsonFormat = options.jsonFormat
+            ? `, format=${(0, parseValue_js_1.default)(options.jsonFormat === "newlineDelimited"
+                ? "newline_delimited"
+                : options.jsonFormat)}`
+            : "";
+        const records = typeof options.records === "boolean"
+            ? `, records=${String(options.records).toUpperCase()}`
+            : "";
+        const columns = columnTypes === null ? "" : `, columns={${columnTypes}}`;
+        return `SELECT ${selectColumns} FROM read_json_auto(${filesAsString}${generalOptions}${columns}${jsonFormat}${records})${conditions}${limit}`;
+    }
+    else if (options.fileType === "parquet" || fileExtension === "parquet") {
+        return `SELECT ${selectColumns} FROM read_parquet(${filesAsString}${includeFilename}${unifyColumns})${conditions}${limit}`;
+    }
+    else if (options.fileType === "excel" || fileExtension === "xlsx") {
+        if (files.length > 1) {
+            throw new Error("For excel files or files with extension .xlsx, you can pass only one file at the time.");
+        }
+        const header = typeof options.header === "boolean"
+            ? `, header=${String(options.header).toUpperCase()}`
+            : ", header=TRUE";
+        const allText = typeof options.allText === "boolean"
+            ? `, all_varchar=${String(options.allText).toUpperCase()}`
+            : "";
+        return `SELECT ${selectColumns} FROM read_xlsx(${(0, parseValue_js_1.default)(files[0])}${options.sheet ? `, sheet=${(0, parseValue_js_1.default)(options.sheet)}` : ""}${header}${allText})${conditions}${limit}`;
+    }
+    else {
+        throw unsupportedFileTypeError(files[0], options.fileType, fileExtension);
+    }
+}
+function unsupportedFileTypeError(file, fileType, fileExtension) {
+    const receivedFileType = JSON.stringify(fileType) ?? String(fileType);
+    return new Error(`loadData() could not determine a supported file type for ${JSON.stringify(file)}. options.fileType: ${receivedFileType}. Detected extension: ${JSON.stringify(`.${fileExtension}`)}. Supported file types: "csv", "dsv", "json", "parquet", and "excel" (for .xlsx files).`);
+}

package/script/methods/loadGeoData.d.ts

@@ -0,0 +1,9 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type LoadGeoDataOptions = {
+    toEPSG4326?: boolean;
+    columns?: string[];
+    conditions?: string;
+};
+export default function loadGeoData(simpleTable: SimpleTable, file: string, options?: LoadGeoDataOptions): void;
+export {};
+//# sourceMappingURL=loadGeoData.d.ts.map

package/script/methods/loadGeoData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/loadGeoData.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,kBAAkB,GAAG;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,kBAAuB,QAYjC"}