@nshiab/simple-data-analysis-core 1.0.0 → 2.0.0

package/script/class/SimpleDB.js

@@ -1,26 +1,81 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
+var _SimpleDB_instances, _SimpleDB_closePromise, _SimpleDB_startPromise, _SimpleDB_initialized, _SimpleDB_setup, _SimpleDB_close;
 Object.defineProperty(exports, "__esModule", { value: true });
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const node_api_1 = require("@duckdb/node-api");
 const runQuery_js_1 = __importDefault(require("../helpers/runQuery.js"));
 const SimpleTable_js_1 = __importDefault(require("./SimpleTable.js"));
 const cleanCache_js_1 = __importDefault(require("../helpers/cleanCache.js"));
-const createDirectory_js_1 = __importDefault(require("../helpers/createDirectory.js"));
 const prettyDuration_js_1 = __importDefault(require("../helpers/prettyDuration.js"));
 const Simple_js_1 = __importDefault(require("./Simple.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const getTableNames_js_1 = __importDefault(require("../methods/getTableNames.js"));
-const cleanPath_js_1 = __importDefault(require("../helpers/cleanPath.js"));
-const getExtension_js_1 = __importDefault(require("../helpers/getExtension.js"));
 const node_fs_1 = require("node:fs");
-const checkVssIndexes_js_1 = __importDefault(require("../helpers/checkVssIndexes.js"));
+const createDirectory_js_1 = __importDefault(require("../helpers/createDirectory.js"));
+const getDbFileType_js_1 = __importDefault(require("../helpers/getDbFileType.js"));
+const getCurrentDatabase_js_1 = __importDefault(require("../helpers/getCurrentDatabase.js"));
+const getDbIndexes_js_1 = __importDefault(require("../helpers/getDbIndexes.js"));
+const readDbMetadata_js_1 = __importDefault(require("../helpers/readDbMetadata.js"));
+const writeDbMetadata_js_1 = __importDefault(require("../helpers/writeDbMetadata.js"));
+const prepareDbExtensions_js_1 = __importDefault(require("../helpers/prepareDbExtensions.js"));
 const setDbProps_js_1 = __importDefault(require("../helpers/setDbProps.js"));
-const writeIndexes_js_1 = __importDefault(require("../helpers/writeIndexes.js"));
-const getName_js_1 = __importDefault(require("../helpers/getName.js"));
-const node_fs_2 = require("node:fs");
+const queryDbFile_js_1 = __importDefault(require("../helpers/queryDbFile.js"));
+const removeTables_js_1 = __importDefault(require("../methods/removeTables.js"));
+const selectTables_js_1 = __importDefault(require("../methods/selectTables.js"));
+const loadDB_js_1 = __importDefault(require("../methods/loadDB.js"));
+const writeDB_js_1 = __importDefault(require("../methods/writeDB.js"));
+const flushAllTables_js_1 = __importStar(require("../helpers/flushAllTables.js"));
+const queueOp_js_1 = require("../helpers/queueOp.js");
+const tableRegistry_js_1 = require("../helpers/tableRegistry.js");
+const formatMissingTables_js_1 = __importDefault(require("../helpers/formatMissingTables.js"));
 /**
  * Manages a DuckDB database instance, providing a simplified interface for database operations.
  *
@@ -28,32 +83,30 @@ const node_fs_2 = require("node:fs");
  * ```ts
  * // Create an in-memory database instance
  * const sdb = new SimpleDB();
- * // Create a new table named "employees"
- * const employees = sdb.newTable("employees");
- * // Load data from a CSV file into the "employees" table
- * await employees.loadData("./employees.csv");
- * // Log the first few rows of the "employees" table to the console
- * await employees.logTable();
+ * // Create a table, load a CSV file, and log its first few rows
+ * const employees = await sdb
+ *   .newTable("employees")
+ *   .loadData("./employees.csv")
+ *   .log();
  * // Close the database connection and clean up resources
- * await sdb.done();
+ * await sdb.close();
  * ```
  *
  * @example
  * ```ts
- * // Create a persistent database instance, saving data to a file
- * // To load an existing database, use the `loadDB` method instead
+ * // Open or create a persistent DuckDB database. Changes persist in this file.
  * const sdb = new SimpleDB({ file: "./my_database.db" });
  * // Perform database operations...
- * // Close the database connection, which saves changes to the specified file
- * await sdb.done();
+ * // Execute pending work, save SDA metadata, and close the database connection
+ * await sdb.close();
  * ```
  *
  * @example
  * ```ts
  * // Create a database instance with custom options
  * const sdb = new SimpleDB({
- *   debug: true, // Enable debugging output
- *   nbRowsToLog: 20 // Set the number of rows to log by default
+ *   logSQL: true, // Log SQL immediately before it runs
+ *   rowsToLog: 20 // Set the number of rows to log by default
  * });
  * ```
  */
@@ -61,21 +114,73 @@ class SimpleDB extends Simple_js_1.default {
     /**
      * Creates a new SimpleDB instance.
      *
+     * DuckDB sessions use UTC so temporal parsing, extraction, and returned
+     * `TIMESTAMP WITH TIME ZONE` strings are consistent across environments.
+     *
+     * Existing tables and SDA index metadata are restored on first use. The
+     * `__sda` schema is reserved for versioned SDA metadata. SQLite files can be
+     * imported with `loadDB()` but cannot be used as the persistent database.
+     *
      * @param options - Configuration options for the SimpleDB instance.
-     * @param options.file - The path to the database file. If not provided, an in-memory database is used.
-     * @param options.overwrite - A flag indicating whether to overwrite the database file if it already exists.
+     * @param options.file - A `.db` or `.duckdb` file to open or create. Defaults to an in-memory database.
+     * @param options.overwrite - If true, replaces an existing file on first use. Defaults to false, which opens the existing file.
+     * @param options.readOnly - Opens an existing DuckDB file read-only. Defaults to false. Requires a file and cannot be combined with overwrite.
      * @param options.logDuration - A flag indicating whether to log the total execution duration.
-     * @param options.nbRowsToLog - The number of rows to display when logging a table.
-     * @param options.nbCharactersToLog - The maximum number of characters to display for text-based cells.
-     * @param options.types - A flag indicating whether to include data types when logging a table.
-     * @param options.cacheVerbose - A flag indicating whether to log verbose cache-related messages.
-     * @param options.debug - A flag indicating whether to log debugging information.
+     * @param options.rowsToLog - The number of rows to display when logging a table.
+     * @param options.charsToLog - The maximum number of characters to display for text-based cells.
+     * @param options.typesToLog - A flag indicating whether to include data types when logging a table.
+     * @param options.cacheVerbose - Whether to log cache hits and misses, code and input changes, TTL status, and cache read/write timing.
+     * @param options.logSQL - A flag indicating whether to log SQL immediately before execution.
+     * @param options.explainSQL - A flag indicating whether to log DuckDB query plans for supported statements.
      * @param options.duckDbCache - A flag indicating whether to use DuckDB's external file cache.
      * @param options.progressBar - A flag indicating whether to display a progress bar for long-running operations.
+     * @param options.memoryLimit - The maximum amount of memory DuckDB is allowed to use (e.g., `'4GB'`). Defaults to 80% of system RAM.
+     * @param options.tempDir - The path to the directory used for temporary files when data exceeds the memory limit (e.g., `'/tmp/duckdb_swap'`). Defaults to `.tmp` for in-memory databases or `<file>.tmp` for file-based databases. Automatically removed when calling `close()`.
      * @category Constructor
+     *
+     * @example
+     * ```ts
+     * const sdb = new SimpleDB({
+     *   logSQL: true,
+     *   explainSQL: true,
+     * });
+     * ```
      */
     constructor(options = {}) {
         super(options);
+        _SimpleDB_instances.add(this);
+        /**
+         * Whether to log each SQL statement immediately before execution.
+         *
+         * @defaultValue `false`
+         * @category Properties
+         * @example
+         * ```ts
+         * const sdb = new SimpleDB({ logSQL: true });
+         * ```
+         */
+        Object.defineProperty(this, "logSQL", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Whether to log DuckDB query plans before supported statements execute.
+         *
+         * @defaultValue `false`
+         * @category Properties
+         * @example
+         * ```ts
+         * const sdb = new SimpleDB({ explainSQL: true });
+         * ```
+         */
+        Object.defineProperty(this, "explainSQL", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * An array of paths to the data sources used in the cache.
          *
@@ -125,19 +230,8 @@ class SimpleDB extends Simple_js_1.default {
             value: void 0
         });
         /**
-         * An array of table instances associated with this database.
-         *
-         * @defaultValue `[]`
-         * @category Properties
-         */
-        Object.defineProperty(this, "tables", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * A flag indicating whether to log verbose cache-related messages.
+         * Whether to log cache hits and misses, code and input changes, TTL status,
+         * and cache read/write timing.
          *
          * @defaultValue `false`
          * @category Properties
@@ -184,6 +278,75 @@ class SimpleDB extends Simple_js_1.default {
             writable: true,
             value: void 0
         });
+        /**
+         * The in-flight flush of the pending chains, if one is running. Concurrent
+         * observers await it instead of reading mid-flush state. This is for
+         * internal use only.
+         *
+         * @defaultValue `null`
+         * @internal
+         */
+        Object.defineProperty(this, "flushPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        _SimpleDB_closePromise.set(this, null);
+        _SimpleDB_startPromise.set(this, null);
+        _SimpleDB_initialized.set(this, false);
+        /**
+         * The database lifecycle state. This is for internal use only.
+         *
+         * @internal
+         */
+        Object.defineProperty(this, "lifecycleState", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * The number of queued operations across all tables, so query execution
+         * points skip the flush entirely when nothing is pending. This is for
+         * internal use only.
+         *
+         * @defaultValue `0`
+         * @internal
+         */
+        Object.defineProperty(this, "pendingCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A counter stamping queued operations with their position in program
+         * order across all tables, so flushes replay them in the order they were
+         * queued. This is for internal use only.
+         *
+         * @defaultValue `0`
+         * @internal
+         */
+        Object.defineProperty(this, "opSequence", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A flag indicating that the spatial extension has been loaded on this
+         * connection, so it's only loaded once. This is for internal use only.
+         *
+         * @defaultValue `false`
+         * @internal
+         */
+        Object.defineProperty(this, "spatialLoaded", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * A flag indicating whether to use DuckDB's external file cache.
          *
@@ -197,7 +360,32 @@ class SimpleDB extends Simple_js_1.default {
             value: void 0
         });
         /**
-         * The path to the database file. If not provided, an in-memory database is used.
+         * The maximum amount of memory DuckDB is allowed to use (e.g., `'4GB'`). Defaults to 80% of system RAM.
+         *
+         * @defaultValue `undefined`
+         * @category Properties
+         */
+        Object.defineProperty(this, "memoryLimit", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * The path to the directory used for temporary files when data exceeds the memory limit (e.g., `'/tmp/duckdb_swap'`). Defaults to `.tmp` for in-memory databases or `<file>.tmp` for file-based databases. Temporary directories are automatically removed when calling `close()`.
+         *
+         * @defaultValue `undefined`
+         * @category Properties
+         */
+        Object.defineProperty(this, "tempDir", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * The path to the persistent DuckDB database, opened or created on first use.
+         * If not provided, an in-memory database is used. Imports and exports do not change this path.
          *
          * @defaultValue `:memory:`
          * @category Properties
@@ -209,7 +397,7 @@ class SimpleDB extends Simple_js_1.default {
             value: void 0
         });
         /**
-         * A flag indicating whether to overwrite the database file if it already exists.
+         * Whether to replace an existing DuckDB file on first use instead of opening it.
          *
          * @defaultValue `false`
          * @category Properties
@@ -220,10 +408,30 @@ class SimpleDB extends Simple_js_1.default {
             writable: true,
             value: void 0
         });
+        /**
+         * Whether the persistent DuckDB file is opened read-only. Requires an existing
+         * file and cannot be combined with `overwrite`. DuckDB rejects writes to it.
+         *
+         * @defaultValue `false`
+         * @category Properties
+         * @example
+         * ```ts
+         * const sdb = new SimpleDB({ file: "./archive.duckdb", readOnly: true });
+         * const table = await sdb.getTable("employees");
+         * await table.log();
+         * await sdb.close();
+         * ```
+         */
+        Object.defineProperty(this, "readOnly", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * The class used to create table instances. Defaults to `SimpleTable`.
          * Override this property when subclassing to ensure all table-creating
-         * methods (e.g., `newTable()`, `cloneTable()`) return instances of your
+         * methods (e.g., `newTable()`, `clone()`) return instances of your
          * custom table class.
          *
          * @defaultValue `SimpleTable`
@@ -254,10 +462,13 @@ class SimpleDB extends Simple_js_1.default {
             value: SimpleTable_js_1.default
         });
         this.file = options.file ?? ":memory:";
+        this.logSQL = options.logSQL ?? false;
+        this.explainSQL = options.explainSQL ?? false;
         this.overwrite = options.overwrite ?? false;
+        this.readOnly = options.readOnly ?? false;
         this.logDuration = options.logDuration ?? false;
         this.tableIncrement = 1;
-        this.tables = [];
+        (0, tableRegistry_js_1.initializeTableRegistry)(this);
         this.cacheSourcesUsed = [];
         this.cacheVerbose = options.cacheVerbose ?? false;
         this.cacheTimeSaved = 0;
@@ -266,6 +477,13 @@ class SimpleDB extends Simple_js_1.default {
         this.duckDbCache = options.duckDbCache === undefined
             ? false
             : options.duckDbCache;
+        this.memoryLimit = options.memoryLimit;
+        this.tempDir = options.tempDir;
+        this.flushPromise = null;
+        this.lifecycleState = "open";
+        this.pendingCount = 0;
+        this.opSequence = 0;
+        this.spatialLoaded = false;
         this.runQuery = runQuery_js_1.default;
         if (this.cacheVerbose || this.logDuration) {
             this.durationStart = Date.now();
@@ -279,52 +497,19 @@ class SimpleDB extends Simple_js_1.default {
      * @category Lifecycle
      */
     async start() {
-        if (this.db === undefined || this.connection === undefined) {
-            if (this.file !== ":memory:") {
-                if ((0, getExtension_js_1.default)(this.file) !== "db") {
-                    throw new Error(`The file extension must be .db. The current file is ${this.file}.`);
-                }
-                if ((0, node_fs_1.existsSync)(this.file) && this.overwrite === false) {
-                    throw new Error(`The file ${this.file} already exists. Set the overwrite option to true to overwrite it. Otherwise, use the loadDB() method to load an existing database with more options.`);
-                }
-                else if ((0, node_fs_1.existsSync)(this.file) && this.overwrite === true) {
-                    (0, node_fs_1.rmSync)(this.file);
-                }
-            }
-            this.db = await node_api_1.DuckDBInstance.create(this.file);
-            this.connection = await this.db.connect();
-            // By default, DuckDB does not compress in-memory databases, so we enable it here.
-            if (this.file === ":memory:") {
-                await this.customQuery("ATTACH OR REPLACE ':memory:' AS memory (COMPRESS);");
-            }
-            if (this.duckDbCache === true) {
-                await this.customQuery("SET enable_external_file_cache=true;");
-            }
-            else if (this.duckDbCache === false) {
-                await this.customQuery("SET enable_external_file_cache=false;");
-            }
-            if (this.progressBar) {
-                await this.customQuery(`SET enable_progress_bar = TRUE; SET progress_bar_time = 0;`);
-            }
+        if (this.lifecycleState === "closed") {
+            throw new Error(`start() cannot use a SimpleDB that is ${this.lifecycleState}.`);
         }
-        return this;
-    }
-    /**
-     * Adds a SimpleTable instance to the internal list of tables.
-     *
-     * @param table - The SimpleTable instance to add.
-     * @internal
-     * @category Table Management
-     */
-    pushTable(table) {
-        const TableClass = this.tableClass;
-        if (!(table instanceof TableClass)) {
-            throw new Error(`The table must be an instance of ${TableClass.name}.`);
+        if (__classPrivateFieldGet(this, _SimpleDB_startPromise, "f") === null) {
+            // The setup queries below must run before any queued operation, so
+            // they must not trigger a flush.
+            __classPrivateFieldSet(this, _SimpleDB_startPromise, (0, flushAllTables_js_1.runExemptFromFlush)(this, async () => {
+                await __classPrivateFieldGet(this, _SimpleDB_instances, "m", _SimpleDB_setup).call(this);
+                __classPrivateFieldSet(this, _SimpleDB_initialized, true, "f");
+                return this;
+            }), "f");
         }
-        if (this.tables.map((t) => t.name).includes(table.name)) {
-            throw new Error(`Table ${table.name} already exists.`);
-        }
-        this.tables.push(table);
+        return await __classPrivateFieldGet(this, _SimpleDB_startPromise, "f");
     }
     /**
      * Creates a new SimpleTable instance within the database.
@@ -336,39 +521,49 @@ class SimpleDB extends Simple_js_1.default {
      * @example
      * ```ts
      * // Create a table with a default name (e.g., "table1", "table2", etc.)
-     * const dataTable = sdb.newTable();
+     * const dataTable = await sdb
+     *   .newTable()
+     *   .loadArray([{ value: 1 }])
+     *   .log();
      * ```
      *
      * @example
      * ```ts
      * // Create a table with a specific name
-     * const employees = sdb.newTable("employees");
+     * const employees = await sdb
+     *   .newTable("employees")
+     *   .loadData("employees.csv")
+     *   .log();
      * ```
      */
     newTable(name) {
+        if (this.lifecycleState !== "open") {
+            throw new Error(`newTable() cannot use a SimpleDB that is ${this.lifecycleState}.`);
+        }
         const TableClass = this.tableClass;
-        // SHOULD MATCH cloneTable
+        // SHOULD MATCH clone
         let table;
         if (typeof name === "string") {
             table = new TableClass(name, this, {
-                debug: this.debug,
-                nbRowsToLog: this.nbRowsToLog,
-                nbCharactersToLog: this.nbCharactersToLog,
-                types: this.types,
+                rowsToLog: this.rowsToLog,
+                charsToLog: this.charsToLog,
+                typesToLog: this.typesToLog,
             });
             table.defaultTableName = false;
         }
         else {
+            while (this.getTables().some((table) => table.name.toLowerCase() === `table${this.tableIncrement}`)) {
+                this.tableIncrement++;
+            }
             table = new TableClass(`table${this.tableIncrement}`, this, {
-                debug: this.debug,
-                nbRowsToLog: this.nbRowsToLog,
-                nbCharactersToLog: this.nbCharactersToLog,
-                types: this.types,
+                rowsToLog: this.rowsToLog,
+                charsToLog: this.charsToLog,
+                typesToLog: this.typesToLog,
             });
             table.defaultTableName = true;
             this.tableIncrement += 1;
         }
-        this.pushTable(table);
+        (0, tableRegistry_js_1.registerTable)(this, table);
         return table;
     }
     /**
@@ -382,22 +577,25 @@ class SimpleDB extends Simple_js_1.default {
      * ```ts
      * // Retrieve the "employees" table
      * const employees = await sdb.getTable("employees");
+     * await employees.log();
      * ```
      */
     async getTable(name) {
-        const table = this.tables.find((t) => t.name === name);
+        await this.start();
+        const tables = this.getTables();
+        const table = tables.find((t) => t.name === name);
         if (table) {
             return await table;
         }
         else {
-            throw new Error(`Table ${name} not found.`);
+            throw new Error((0, formatMissingTables_js_1.default)("getTable()", [name], tables.map((availableTable) => availableTable.name)));
         }
     }
     /**
      * Removes one or more tables from the database.
      *
-     * @param tables - A single table or an array of tables to remove, specified by name or as SimpleTable instances.
-     * @returns A promise that resolves when the tables have been removed.
+     * @param tables - A single table or an array of tables to remove, specified by name or as SimpleTable instances. Pass `"all"` to remove all tables.
+     * @returns A promise that resolves to the database, so methods can be chained.
      * @category Table Management
      *
      * @example
@@ -419,22 +617,22 @@ class SimpleDB extends Simple_js_1.default {
      * // ... load data ...
      * await sdb.removeTables(employeesTable);
      * ```
+     *
+     * @example
+     * ```ts
+     * // Remove all tables
+     * await sdb.removeTables("all");
+     * ```
      */
     async removeTables(tables) {
-        const tablesToBeRemoved = Array.isArray(tables) ? tables : [tables];
-        await (0, queryDB_js_1.default)(this, tablesToBeRemoved.map((d) => `DROP TABLE "${d instanceof SimpleTable_js_1.default ? d.name : d}";`).join("\n"), (0, mergeOptions_js_1.default)(this, {
-            table: null,
-            method: "removeTable()",
-            parameters: {},
-        }));
-        const tablesNamesToBeRemoved = tablesToBeRemoved.map((t) => t instanceof SimpleTable_js_1.default ? t.name : t);
-        this.tables = this.tables.filter((t) => !tablesNamesToBeRemoved.includes(t.name));
+        await (0, removeTables_js_1.default)(this, tables);
+        return this;
     }
     /**
      * Selects one or more tables to keep in the database, removing all others.
      *
      * @param tables - A single table or an array of tables to select, specified by name or as SimpleTable instances.
-     * @returns A promise that resolves when the tables have been selected.
+     * @returns A promise that resolves to the database, so methods can be chained.
      * @category Table Management
      *
      * @example
@@ -458,20 +656,8 @@ class SimpleDB extends Simple_js_1.default {
      * ```
      */
     async selectTables(tables) {
-        const tablesToBeSelected = (Array.isArray(tables) ? tables : [tables]).map((t) => t instanceof SimpleTable_js_1.default ? t.name : t);
-        for (const table of tablesToBeSelected) {
-            if (!(await this.hasTable(table))) {
-                throw new Error(`Table ${table} not found.`);
-            }
-        }
-        const tablesToBeRemoved = this.tables.filter((t) => !tablesToBeSelected.includes(t.name));
-        await (0, queryDB_js_1.default)(this, tablesToBeRemoved.map((d) => `DROP TABLE "${d instanceof SimpleTable_js_1.default ? d.name : d}";`).join("\n"), (0, mergeOptions_js_1.default)(this, {
-            table: null,
-            method: "removeTable()",
-            parameters: {},
-        }));
-        const tablesNamesToBeRemoved = tablesToBeRemoved.map((t) => t instanceof SimpleTable_js_1.default ? t.name : t);
-        this.tables = this.tables.filter((t) => !tablesNamesToBeRemoved.includes(t.name));
+        await (0, selectTables_js_1.default)(this, tables);
+        return this;
     }
     /**
      * Returns an array of all table names in the database, sorted alphabetically.
@@ -492,7 +678,7 @@ class SimpleDB extends Simple_js_1.default {
     /**
      * Logs the names of all tables in the database to the console, sorted alphabetically.
      *
-     * @returns A promise that resolves when the table names have been logged.
+     * @returns A promise that resolves to the database, so methods can be chained.
      * @category Table Management
      *
      * @example
@@ -510,21 +696,25 @@ class SimpleDB extends Simple_js_1.default {
         else {
             console.log(`\nSimpleDB - No tables found.`);
         }
+        return this;
     }
     /**
      * Returns an array of all SimpleTable instances in the database.
      *
-     * @returns A promise that resolves to an array of SimpleTable instances.
+     * The returned array is a snapshot and cannot mutate the database's internal
+     * table registry.
+     *
+     * @returns A read-only array of SimpleTable instances.
      * @category Table Management
      *
      * @example
      * ```ts
      * // Get all SimpleTable instances
-     * const tables = await sdb.getTables();
+     * const tables = sdb.getTables();
      * ```
      */
-    async getTables() {
-        return await this.tables;
+    getTables() {
+        return (0, tableRegistry_js_1.listRegisteredTables)(this);
     }
     /**
      * Checks if a table exists in the database.
@@ -568,7 +758,7 @@ class SimpleDB extends Simple_js_1.default {
      */
     async getExtensions() {
         return (await (0, queryDB_js_1.default)(this, `FROM duckdb_extensions();`, (0, mergeOptions_js_1.default)(this, {
-            returnDataFrom: "query",
+            returnData: true,
             table: null,
             method: "getExtensions()",
             parameters: {},
@@ -576,15 +766,20 @@ class SimpleDB extends Simple_js_1.default {
     }
     /**
      * Executes a custom SQL query directly against the DuckDB instance.
+     * Queries run in UTC. When data is returned, temporal values use the same
+     * JavaScript representations as `SimpleTable.getData()`.
      *
-     * If you want to force the returned data to match the types of the columns, you can use the `types` option.
+     * `customQuery()` bypasses the dependency and table-generation tracking used
+     * by `SimpleTable.cache()`. Reading or changing a table with `customQuery()`
+     * can therefore cause `cache()` to return stale data. Include a value that
+     * identifies the custom query's dependencies in the cache's `options.inputs`
+     * (such as a table content hash), or use tracked `SimpleTable` methods.
      *
      * @param query - The SQL query string to execute.
      * @param options - Configuration options for the query.
-     * @param options.returnDataFrom - Specifies whether to return data from the query. Can be `"query"` to return data or `"none"` (default) to not return data.
+     * @param options.returnData - If `true`, the query result is returned. Defaults to `false`.
      * @param options.table - The name of the table associated with the query, primarily used for debugging and logging.
-     * @param options.types - An optional object specifying data types for the query parameters.
-     * @returns A promise that resolves to the query result as an array of objects if `returnDataFrom` is `"query"`, otherwise `null`.
+     * @returns A promise that resolves to the query result as an array of objects if `returnData` is `true`, otherwise `null`.
      * @category DuckDB
      *
      * @example
@@ -598,29 +793,33 @@ class SimpleDB extends Simple_js_1.default {
      * // Execute a query and return the results
      * const youngEmployees = await sdb.customQuery(
      *   "SELECT * FROM employees WHERE age < 30",
-     *   { returnDataFrom: "query" }
+     *   { returnData: true }
      * );
      * console.log(youngEmployees);
      * ```
      */
     async customQuery(query, options = {}) {
         return await (0, queryDB_js_1.default)(this, query, (0, mergeOptions_js_1.default)(this, {
-            returnDataFrom: options.returnDataFrom,
+            returnData: options.returnData ?? false,
             table: options.table ?? null,
             method: "customQuery()",
             parameters: { query, options },
-            types: options.types,
         }));
     }
     /**
-     * Loads a database from a specified file into the current SimpleDB instance.
-     * Supported file types are `.db` (DuckDB) and `.sqlite` (SQLite).
-     *
-     * @param file - The absolute path to the database file (e.g., "./my_database.db").
-     * @param options - Configuration options for loading the database.
-     * @param options.name - The name to assign to the loaded database within the DuckDB instance. Defaults to the file name without extension.
-     * @param options.detach - If `true` (default), the database is detached after loading its contents into memory. If `false`, the database remains attached.
-     * @returns A promise that resolves when the database has been loaded.
+     * Imports a copy of a `.db` or `.duckdb` (DuckDB) or `.sqlite` (SQLite)
+     * file into the current database. The source is opened read-only and detached
+     * after importing; subsequent transformations do not modify the source file.
+     * Imports work with in-memory and writable persistent databases. Existing
+     * table-name conflicts are rejected and a failed copy is rolled back.
+     *
+     * DuckDB files restore embedded SDA index definitions when present. SQLite
+     * imports copy data without SDA index metadata. The `__sda` schema is reserved
+     * for SDA metadata.
+     * To edit an existing DuckDB file in place, use `new SimpleDB({ file })`.
+     *
+     * @param file - The relative or absolute path to the database file.
+     * @returns A promise that resolves to the database, so methods can be chained.
      * @category File Operations
      *
      * @example
@@ -631,85 +830,42 @@ class SimpleDB extends Simple_js_1.default {
      *
      * @example
      * ```ts
-     * // Load a SQLite database file and keep it attached
-     * await sdb.loadDB("./my_database.sqlite", { detach: false });
+     * // Import SQLite tables without modifying the original file
+     * await sdb.loadDB("./my_database.sqlite");
      * ```
      *
      * @example
      * ```ts
-     * // Load a database with a custom name
-     * await sdb.loadDB("./archive.db", { name: "archive_db" });
+     * // Import a copy into a persistent DuckDB database
+     * const sdb = new SimpleDB({ file: "./analysis.duckdb" });
+     * await sdb.loadDB("./archive.db");
+     * await sdb.close();
      * ```
      */
-    async loadDB(file, options = {}) {
-        const name = options.name ?? "my_database";
-        const detach = options.detach ?? true;
-        if (!(0, node_fs_1.existsSync)(file)) {
-            throw new Error(`The file ${file} does not exist.`);
-        }
-        const extension = (0, getExtension_js_1.default)(file);
-        const allIndexesFile = `${file.replace(`.${extension}`, "")}_indexes.json`;
-        const vssIndex = (0, checkVssIndexes_js_1.default)(allIndexesFile);
-        if (vssIndex) {
-            await this.customQuery(`INSTALL vss; LOAD vss;`);
-        }
-        if (extension === "db") {
-            if (detach) {
-                await (0, queryDB_js_1.default)(this, `ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${name};
-COPY FROM DATABASE ${name} TO memory;
-DETACH ${name};`, (0, mergeOptions_js_1.default)(this, {
-                    returnDataFrom: "none",
-                    table: null,
-                    method: "loadDB()",
-                    parameters: {},
-                }));
-            }
-            else {
-                await (0, queryDB_js_1.default)(this, `ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${name};
-          USE ${name};`, (0, mergeOptions_js_1.default)(this, {
-                    returnDataFrom: "none",
-                    table: null,
-                    method: "loadDB()",
-                    parameters: {},
-                }));
-            }
-        }
-        else if (extension === "sqlite") {
-            if (detach) {
-                await (0, queryDB_js_1.default)(this, `INSTALL sqlite; LOAD sqlite;
-        ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${name} (TYPE SQLITE);
-COPY FROM DATABASE ${name} TO memory;
-DETACH ${name};`, (0, mergeOptions_js_1.default)(this, {
-                    returnDataFrom: "none",
-                    table: null,
-                    method: "loadDB()",
-                    parameters: {},
-                }));
-            }
-            else {
-                await (0, queryDB_js_1.default)(this, `INSTALL sqlite; LOAD sqlite;
-        ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${name} (TYPE SQLITE);
-        USE ${name};`, (0, mergeOptions_js_1.default)(this, {
-                    returnDataFrom: "none",
-                    table: null,
-                    method: "loadDB()",
-                    parameters: {},
-                }));
-            }
-        }
-        else {
-            throw new Error(`The extension ${extension} is not supported. Please use .db or .sqlite instead.`);
-        }
-        await (0, setDbProps_js_1.default)(this, allIndexesFile);
+    async loadDB(file) {
+        await (0, loadDB_js_1.default)(this, file);
+        return this;
     }
     /**
-     * Writes the current state of the database to a specified file.
-     * Supported output file types are `.db` (DuckDB) and `.sqlite` (SQLite).
+     * Exports a snapshot of the current database after executing pending work.
+     * Does not change the working database or where subsequent changes persist.
+     * DuckDB outputs (`.db` and `.duckdb`) preserve database objects and embed SDA
+     * index definitions in the reserved `__sda` schema.
      *
-     * @param file - The absolute path to the output file (e.g., "./my_exported_database.db").
+     * SQLite output (`.sqlite`) materializes main-schema tables and views as
+     * tables. It does not preserve DuckDB schemas, indexes, constraints, or SDA
+     * metadata, and SQLite type conversion may lose type information. Unsupported
+     * conversions fail without replacing an existing destination.
+     *
+     * Existing files require explicit overwrite permission. The completed export
+     * is published only after its connection is detached. Database files attached
+     * to this instance, directories, and symbolic links cannot be replaced.
+     *
+     * @param file - The relative or absolute path to the output file.
      * @param options - Configuration options for writing the database.
-     * @param options.noMetaData - If `true`, metadata files (indexes) are not created alongside the database file. Defaults to `false`.
-     * @returns A promise that resolves when the database has been written to the file.
+     * @param options.overwrite - If true, permits atomic replacement of an existing output file. Defaults to false.
+     * @param options.metadata - If false, omits logical SDA index definitions from DuckDB exports. Defaults to true. Does not control physical indexes; SQLite exports never include SDA metadata.
+     * @returns A promise that resolves to the database, so methods can be chained.
      * @category File Operations
      *
      * @example
@@ -720,92 +876,202 @@ DETACH ${name};`, (0, mergeOptions_js_1.default)(this, {
      *
      * @example
      * ```ts
-     * // Write the current database to a SQLite file without metadata
-     * await sdb.writeDB("./my_exported_database.sqlite", { noMetaData: true });
+     * // Explicitly replace an earlier snapshot
+     * await sdb.writeDB("./my_exported_database.duckdb", { overwrite: true });
+     * ```
+     *
+     * @example
+     * ```ts
+     * // Export table data for use with SQLite
+     * await sdb.writeDB("./my_exported_database.sqlite");
      * ```
      */
     async writeDB(file, options = {}) {
-        const noMetaData = options.noMetaData ?? false;
-        if ((0, node_fs_1.existsSync)(file)) {
-            (0, node_fs_1.rmSync)(file);
-        }
-        (0, createDirectory_js_1.default)(file);
-        const extension = (0, getExtension_js_1.default)(file);
-        if (!noMetaData) {
-            (0, writeIndexes_js_1.default)(this, extension, file);
-        }
-        const name = (0, getName_js_1.default)(file);
-        if (extension === "db") {
-            await (0, queryDB_js_1.default)(this, `ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${name};
-COPY FROM DATABASE ${(0, getName_js_1.default)(this.file)} TO ${name};
-DETACH ${name};`, (0, mergeOptions_js_1.default)(this, {
-                returnDataFrom: "none",
-                table: null,
-                method: "writeDB()",
-                parameters: {},
-            }));
-        }
-        else if (extension === "sqlite") {
-            await (0, queryDB_js_1.default)(this, `INSTALL sqlite; LOAD sqlite;
-        ATTACH '${(0, cleanPath_js_1.default)(file)}' AS ${name} (TYPE SQLITE);
-COPY FROM DATABASE ${(0, getName_js_1.default)(this.file)} TO ${name};
-DETACH ${name};`, (0, mergeOptions_js_1.default)(this, {
-                returnDataFrom: "none",
-                table: null,
-                method: "writeDB()",
-                parameters: {},
-            }));
-        }
-        else {
-            throw new Error(`The extension ${extension} is not supported. Please use .db or .sqlite instead.`);
+        await (0, writeDB_js_1.default)(this, file, options);
+        return this;
+    }
+    /**
+     * Executes all queued methods across every table in the database. Sync
+     * builder methods (like `filter()` or `convert()`) only queue their
+     * operation; execution happens when an async observer method (like
+     * `getData()`, `log()`, or `writeData()`) is awaited. Use `run()` when
+     * your script ends in pure mutations with nothing to observe and you want
+     * the work done now.
+     *
+     * The whole database is flushed in program order, so operations on
+     * different tables execute exactly in the order they were queued. This is
+     * the database-level counterpart to `SimpleTable.run()`.
+     *
+     * @returns A promise that resolves to the SimpleDB instance once the queued methods have been executed.
+     * @category Lifecycle
+     *
+     * @example
+     * ```ts
+     * // Nothing is observed after the mutations, so run() executes them.
+     * table1.loadData("data1.csv").convert({ price: "number" });
+     * table2.loadData("data2.csv").filter(`price > 0`);
+     * await sdb.run();
+     * ```
+     */
+    async run() {
+        if (this.lifecycleState !== "open") {
+            throw new Error(`run() cannot use a SimpleDB that is ${this.lifecycleState}.`);
         }
+        await (0, flushAllTables_js_1.default)(this);
+        return this;
     }
     /**
-     * Frees up memory by closing the database connection and instance, and cleans up the cache.
-     * If the database is file-based, it also compacts the database file to optimize storage.
+     * Executes pending transformations, saves SDA metadata for writable persistent
+     * databases, and closes the connection and instance. Also cleans up temporary
+     * files and the cache. Does not copy, compact, or replace the database file.
      *
      * @returns A promise that resolves to the SimpleDB instance after cleanup.
+     * @throws An error after cleanup when pending execution or cleanup fails.
      * @category Lifecycle
      *
      * @example
      * ```ts
-     * // Close the database and clean up resources
-     * await sdb.done();
+     * // close() executes queued transformations before cleaning up resources.
+     * table.loadData("data.csv").convert({ price: "number" });
+     * await sdb.close();
      * ```
      */
-    async done() {
-        if (this.file !== ":memory:") {
-            await this.customQuery("CHECKPOINT;");
-            // To make sure the files will have the proper names.
-            (0, writeIndexes_js_1.default)(this, (0, getExtension_js_1.default)(this.file), this.file);
-            await this.writeDB(this.file.replace(".db", "_compacted.db"), {
-                noMetaData: true,
-            });
+    async close() {
+        if (__classPrivateFieldGet(this, _SimpleDB_closePromise, "f") !== null) {
+            return await __classPrivateFieldGet(this, _SimpleDB_closePromise, "f");
+        }
+        this.lifecycleState = "closing";
+        const closePromise = __classPrivateFieldGet(this, _SimpleDB_instances, "m", _SimpleDB_close).call(this);
+        __classPrivateFieldSet(this, _SimpleDB_closePromise, closePromise, "f");
+        return await closePromise;
+    }
+}
+_SimpleDB_closePromise = new WeakMap(), _SimpleDB_startPromise = new WeakMap(), _SimpleDB_initialized = new WeakMap(), _SimpleDB_instances = new WeakSet(), _SimpleDB_setup = async function _SimpleDB_setup() {
+    if (this.readOnly && (this.file === ":memory:" || this.overwrite)) {
+        throw new Error("readOnly requires an existing database file and cannot be combined with overwrite.");
+    }
+    if (this.file !== ":memory:") {
+        if ((0, getDbFileType_js_1.default)(this.file) !== "duckdb") {
+            throw new Error("Persistent databases must use .db or .duckdb. Use loadDB() to import SQLite files.");
+        }
+        if (this.readOnly && !(0, node_fs_1.existsSync)(this.file)) {
+            throw new Error(`The file ${this.file} does not exist.`);
+        }
+        if ((0, node_fs_1.existsSync)(this.file) && this.overwrite) {
             (0, node_fs_1.rmSync)(this.file);
-            (0, node_fs_2.renameSync)(this.file.replace(".db", "_compacted.db"), this.file);
+            // A WAL belongs to the database being explicitly replaced.
+            (0, node_fs_1.rmSync)(`${this.file}.wal`, { force: true });
         }
-        if (this.db instanceof node_api_1.DuckDBInstance) {
+        if (!this.readOnly)
+            (0, createDirectory_js_1.default)(this.file);
+    }
+    this.db = await node_api_1.DuckDBInstance.create(this.file, this.readOnly ? { access_mode: "READ_ONLY" } : {});
+    this.connection = await this.db.connect();
+    // Keep temporal parsing, extraction, and serialization deterministic
+    // across machines. DuckDB otherwise inherits the process timezone.
+    await this.customQuery("SET TimeZone = 'UTC';");
+    // By default, DuckDB does not compress in-memory databases, so we enable it here.
+    if (this.file === ":memory:") {
+        await this.customQuery(`ATTACH OR REPLACE ':memory:' AS ${(0, quoteIdentifier_js_1.default)("memory")} (COMPRESS);`);
+    }
+    if (this.duckDbCache === true) {
+        await this.customQuery("SET enable_external_file_cache=true;");
+    }
+    else if (this.duckDbCache === false) {
+        await this.customQuery("SET enable_external_file_cache=false;");
+    }
+    if (this.progressBar) {
+        await this.customQuery(`SET enable_progress_bar = TRUE; SET progress_bar_time = 0;`);
+    }
+    if (this.memoryLimit !== undefined) {
+        await this.customQuery(`SET memory_limit = '${this.memoryLimit}';`);
+    }
+    if (this.tempDir !== undefined) {
+        await this.customQuery(`SET temp_directory = '${this.tempDir}';`);
+    }
+    if (this.file !== ":memory:") {
+        const database = await (0, getCurrentDatabase_js_1.default)(this);
+        const indexes = await (0, readDbMetadata_js_1.default)(this, database);
+        await (0, prepareDbExtensions_js_1.default)(this, database, indexes);
+        const existingNames = new Set((await this.getTableNames()).map((name) => name.toLowerCase()));
+        const conflicts = this.getTables().filter((table) => existingNames.has(table.name.toLowerCase()));
+        if (conflicts.length > 0) {
+            throw new Error(`Tables already exist in ${this.file}: ${conflicts.map((table) => table.name).join(", ")}. Use getTable() to access them.`);
+        }
+        await (0, setDbProps_js_1.default)(this, indexes);
+    }
+}, _SimpleDB_close = async function _SimpleDB_close() {
+    let operationError;
+    try {
+        await (0, flushAllTables_js_1.default)(this);
+        if (this.file !== ":memory:" && !this.readOnly && __classPrivateFieldGet(this, _SimpleDB_initialized, "f")) {
+            const database = await (0, getCurrentDatabase_js_1.default)(this);
+            await (0, readDbMetadata_js_1.default)(this, database);
+            await (0, writeDbMetadata_js_1.default)(this, database, (0, getDbIndexes_js_1.default)(this));
+            await (0, queryDbFile_js_1.default)(this, `CHECKPOINT ${(0, quoteIdentifier_js_1.default)(database)};`);
+        }
+    }
+    catch (error) {
+        operationError = error;
+        // A failed flush can requeue work belonging to other tables. The
+        // connection is about to close, so release those captured values.
+        (0, queueOp_js_1.discardAllPending)(this);
+    }
+    const cleanupErrors = [];
+    if (this.db instanceof node_api_1.DuckDBInstance) {
+        try {
             this.connection.closeSync();
+        }
+        catch (error) {
+            cleanupErrors.push(error);
+        }
+        try {
             this.db.closeSync();
         }
+        catch (error) {
+            cleanupErrors.push(error);
+        }
+    }
+    try {
+        const tmpDir = this.tempDir ??
+            (this.file === ":memory:" ? ".tmp" : `${this.file}.tmp`);
+        if ((0, node_fs_1.existsSync)(tmpDir)) {
+            (0, node_fs_1.rmSync)(tmpDir, { recursive: true });
+        }
         (0, cleanCache_js_1.default)(this);
-        if (typeof this.durationStart === "number") {
-            let string = (0, prettyDuration_js_1.default)(this.durationStart, {
-                prefix: "\n\nSimpleDB - Done in ",
-            });
-            if (this.cacheTimeSaved > 0) {
-                string += ` / ${(0, prettyDuration_js_1.default)(0, {
-                    end: this.cacheTimeSaved,
-                })} saved by using the cache`;
-            }
-            if (this.cacheTimeWriting > 0) {
-                string += ` / ${(0, prettyDuration_js_1.default)(0, {
-                    end: this.cacheTimeWriting,
-                })} spent writing the cache`;
-            }
-            console.log(`${string}\n`);
+    }
+    catch (error) {
+        cleanupErrors.push(error);
+    }
+    this.lifecycleState = "closed";
+    if (operationError !== undefined && cleanupErrors.length > 0) {
+        throw new AggregateError([operationError, ...cleanupErrors], "Pending execution and database cleanup both failed.", { cause: operationError });
+    }
+    if (operationError !== undefined) {
+        throw operationError;
+    }
+    if (cleanupErrors.length === 1) {
+        throw cleanupErrors[0];
+    }
+    if (cleanupErrors.length > 1) {
+        throw new AggregateError(cleanupErrors, "Multiple errors occurred while cleaning up the database.", { cause: cleanupErrors[0] });
+    }
+    if (typeof this.durationStart === "number") {
+        let string = (0, prettyDuration_js_1.default)(this.durationStart, {
+            prefix: "\n\nSimpleDB ran for ",
+        });
+        if (this.cacheTimeSaved > 0) {
+            string += ` / ${(0, prettyDuration_js_1.default)(0, {
+                end: this.cacheTimeSaved,
+            })} saved by using the cache`;
         }
-        return await this;
+        if (this.cacheTimeWriting > 0) {
+            string += ` / ${(0, prettyDuration_js_1.default)(0, {
+                end: this.cacheTimeWriting,
+            })} spent writing the cache`;
+        }
+        console.log(`${string}\n`);
     }
-}
+    return await this;
+};
 exports.default = SimpleDB;