@malloydata/db-duckdb 0.0.390 → 0.0.392

package/dist/duckdb_common.js

@@ -32,6 +32,14 @@ const unquoteName = (name) => {
     }
     return name;
 };
+/**
+ * A bare or schema-qualified SQL identifier — `name` or `schema.name`. Anything
+ * else (file paths with dashes, dots-as-extensions, slashes, URL schemes,
+ * globs) is treated as a file-path string by `fetchTableSchema` and wrapped
+ * in single quotes so DuckDB sees it as a string literal rather than parsing
+ * it as a SQL identifier.
+ */
+const SQL_IDENTIFIER_CHAIN = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$/;
 class DuckDBCommon extends connection_1.BaseConnection {
     get dialectName() {
         return this.dialect.name;
@@ -123,9 +131,9 @@ class DuckDBCommon extends connection_1.BaseConnection {
             connection: this.name,
             fields: [],
         };
-        const quotedTablePath = tablePath.match(/[:*/]/)
-            ? `'${tablePath}'`
-            : tablePath;
+        const quotedTablePath = SQL_IDENTIFIER_CHAIN.test(tablePath)
+            ? tablePath
+            : `'${tablePath}'`;
         const infoQuery = `DESCRIBE SELECT * FROM ${quotedTablePath}`;
         await this.schemaFromQuery(infoQuery, structDef);
         return structDef;

package/dist/duckdb_config.d.ts

@@ -34,6 +34,8 @@ export interface NormalizedDuckDBConfig {
     motherDuckToken?: string;
     additionalExtensions: string[];
     setupSQL?: string;
+    shareable: boolean;
+    effectiveShareable: boolean;
 }
 export declare class DuckDBConfigValidationError extends Error {
     constructor(message: string);

package/dist/duckdb_config.js

@@ -60,7 +60,7 @@ const REQUIRED_BASELINE_EXTENSIONS = ['icu', 'json'];
 const DERIVED_TEMP_DIRECTORY_NAME = '.tmp';
 const DERIVED_SECRET_DIRECTORY_NAME = '.duckdb-secrets';
 function normalizeDuckDBConfig(config) {
-    var _a, _b, _c, _d;
+    var _a, _b, _c, _d, _e;
     const securityPolicy = parseSecurityPolicy(config['securityPolicy']);
     if (securityPolicy === 'sandboxed' && !pathSecurity.isPosixHost()) {
         throw new DuckDBConfigValidationError('securityPolicy "sandboxed" is only supported on POSIX hosts');
@@ -70,6 +70,7 @@ function normalizeDuckDBConfig(config) {
     const rawSetupSQL = normalizeOptionalText(readOptionalString(config, 'setupSQL'));
     const rawMotherDuckToken = normalizeOptionalText(readOptionalString(config, 'motherDuckToken'));
     const readOnly = (_c = readOptionalBoolean(config, 'readOnly')) !== null && _c !== void 0 ? _c : false;
+    const shareable = (_d = readOptionalBoolean(config, 'shareable')) !== null && _d !== void 0 ? _d : false;
     const additionalExtensions = normalizeExtensions(config['additionalExtensions'], 'additionalExtensions');
     const enableExternalAccess = readOptionalBoolean(config, 'enableExternalAccess');
     const lockConfiguration = readOptionalBoolean(config, 'lockConfiguration');
@@ -175,7 +176,7 @@ function normalizeDuckDBConfig(config) {
         ? undefined
         : canonicalizeConfigPath(rawExtensionDirectory, 'extensionDirectory');
     const secretDirectory = deriveRestrictedSecretDirectory({
-        requiresSecretNeutralization: (_d = safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.requiresSecretNeutralization) !== null && _d !== void 0 ? _d : false,
+        requiresSecretNeutralization: (_e = safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.requiresSecretNeutralization) !== null && _e !== void 0 ? _e : false,
         tempDirectory,
         workingDirectory,
     });
@@ -206,6 +207,10 @@ function normalizeDuckDBConfig(config) {
         motherDuckToken: rawMotherDuckToken,
         additionalExtensions,
         setupSQL: rawSetupSQL,
+        shareable,
+        effectiveShareable: shareable &&
+            databasePath !== ':memory:' &&
+            !isLikelyRemoteDatabasePath(databasePath),
     };
 }
 function buildDuckDBShareKey(config) {
@@ -226,7 +231,7 @@ function buildDuckDBShareKey(config) {
         ? ''
         : String(config.allowUnsignedExtensions), config.tempFileEncryption === undefined
         ? ''
-        : String(config.tempFileEncryption), config.threads === undefined ? '' : String(config.threads), (_c = config.memoryLimit) !== null && _c !== void 0 ? _c : '', (_d = config.tempDirectory) !== null && _d !== void 0 ? _d : '', (_e = config.workingDirectory) !== null && _e !== void 0 ? _e : '', ...[...config.additionalExtensions].sort(), (_f = config.extensionDirectory) !== null && _f !== void 0 ? _f : '', (_g = config.motherDuckToken) !== null && _g !== void 0 ? _g : '');
+        : String(config.tempFileEncryption), config.threads === undefined ? '' : String(config.threads), (_c = config.memoryLimit) !== null && _c !== void 0 ? _c : '', (_d = config.tempDirectory) !== null && _d !== void 0 ? _d : '', (_e = config.workingDirectory) !== null && _e !== void 0 ? _e : '', ...[...config.additionalExtensions].sort(), (_f = config.extensionDirectory) !== null && _f !== void 0 ? _f : '', (_g = config.motherDuckToken) !== null && _g !== void 0 ? _g : '', String(config.effectiveShareable));
 }
 function sqlStringLiteral(value) {
     return `'${value.replace(/'/g, "''")}'`;

package/dist/duckdb_connection.d.ts

@@ -22,6 +22,7 @@ export interface DuckDBConnectionOptions extends ConnectionConfig {
     memoryLimit?: string;
     tempDirectory?: string;
     extensionDirectory?: string;
+    shareable?: boolean;
 }
 interface ActiveDB {
     instance: DuckDBInstance;
@@ -44,6 +45,28 @@ export declare class DuckDBConnection extends DuckDBCommon {
      * from reversible idle.
      */
     protected closed: boolean;
+    /**
+     * Shareable mode only. The DuckDBInstance backing this connection's
+     * `:memory:` primary database. Not shared via `activeDBs` because each
+     * shareable connection independently ATTACHes/DETACHes the real file —
+     * there is nothing to pool. Closed in `close()`.
+     */
+    private ownedInstance;
+    /**
+     * Shareable mode only. True between a successful ATTACH (in
+     * `setupOnce()`) and the matching DETACH (in `idle()` / `close()`).
+     */
+    private attached;
+    /**
+     * Shareable mode only. True after the first successful run of the
+     * once-per-connection portion of `setupOnce()` (baseline `SET`s,
+     * extension `LOAD`s, user `setupSQL`, `lockConfiguration`). Those run
+     * against the persistent `:memory:` primary and must NOT re-run on
+     * later wake-ups: user `setupSQL` may be non-idempotent, and any later
+     * `SET` would fail under `lockConfiguration=true`. Wake-ups still
+     * re-ATTACH the real file because that's the bit DETACH released.
+     */
+    private connectionSetupRan;
     static activeDBs: Record<string, ActiveDB>;
     constructor(options: DuckDBConnectionOptions, queryOptions?: QueryOptionsReader);
     constructor(name: string, databasePath?: string, workingDirectory?: string, queryOptions?: QueryOptionsReader);
@@ -51,6 +74,14 @@ export declare class DuckDBConnection extends DuckDBCommon {
     private init;
     protected setup(): Promise<void>;
     private setupOnce;
+    /**
+     * Shareable mode: ATTACH the real database file into the in-memory
+     * primary and `USE` it, so unqualified table refs resolve into the
+     * attached file. Idempotent — guarded by `this.attached`. Runs after
+     * `applyFinalBaseline()` so the corresponding `SET allowed_directories`
+     * (sandboxed mode) has already been applied.
+     */
+    private attachIfShareable;
     private buildInstanceOptions;
     private shouldApplyEnableExternalAccessAtOpenTime;
     private applyFinalBaseline;
@@ -63,6 +94,7 @@ export declare class DuckDBConnection extends DuckDBCommon {
     runSQLStream(sql: string, { rowLimit, abortSignal }?: RunSQLOptions): AsyncIterableIterator<QueryRecord>;
     close(): Promise<void>;
     idle(): Promise<void>;
+    private detachShareableFile;
     /**
      * Drop our entry from the shared `activeDBs` bookkeeping. If we were
      * the last sharer, close the underlying `DuckDBInstance`.

package/dist/duckdb_connection.js

@@ -31,6 +31,7 @@ const node_api_1 = require("@duckdb/node-api");
 const malloy_1 = require("@malloydata/malloy");
 const package_json_1 = __importDefault(require("@malloydata/malloy/package.json"));
 const duckdb_config_1 = require("./duckdb_config");
+const SHAREABLE_ATTACH_ALIAS = 'malloy_db';
 class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     constructor(arg, arg2, workingDirectory, queryOptions) {
         var _a;
@@ -42,6 +43,28 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
          * from reversible idle.
          */
         this.closed = false;
+        /**
+         * Shareable mode only. The DuckDBInstance backing this connection's
+         * `:memory:` primary database. Not shared via `activeDBs` because each
+         * shareable connection independently ATTACHes/DETACHes the real file —
+         * there is nothing to pool. Closed in `close()`.
+         */
+        this.ownedInstance = null;
+        /**
+         * Shareable mode only. True between a successful ATTACH (in
+         * `setupOnce()`) and the matching DETACH (in `idle()` / `close()`).
+         */
+        this.attached = false;
+        /**
+         * Shareable mode only. True after the first successful run of the
+         * once-per-connection portion of `setupOnce()` (baseline `SET`s,
+         * extension `LOAD`s, user `setupSQL`, `lockConfiguration`). Those run
+         * against the persistent `:memory:` primary and must NOT re-run on
+         * later wake-ups: user `setupSQL` may be non-idempotent, and any later
+         * `SET` would fail under `lockConfiguration=true`. Wake-ups still
+         * re-ATTACH the real file because that's the bit DETACH released.
+         */
+        this.connectionSetupRan = false;
         const options = typeof arg === 'string'
             ? buildLegacyOptions(arg, arg2, workingDirectory)
             : arg;
@@ -72,6 +95,15 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     }
     async init() {
         try {
+            if (this.normalized.effectiveShareable) {
+                // Shareable mode: own private :memory: primary, no activeDBs
+                // sharing. The real database file is opened lazily via ATTACH in
+                // setupOnce() and released via DETACH in idle().
+                const instance = await node_api_1.DuckDBInstance.create(':memory:', this.buildInstanceOptions({ forShareablePrimary: true }));
+                this.connection = await instance.connect();
+                this.ownedInstance = instance;
+                return;
+            }
             const cached = DuckDBConnection.activeDBs[this.shareKey];
             if (cached) {
                 this.connection = await cached.instance.connect();
@@ -116,24 +148,65 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         await this.isSetup;
     }
     async setupOnce() {
-        await this.applyFinalBaseline();
-        if (this.normalized.setupSQL) {
-            for (const statement of splitSetupSQL(this.normalized.setupSQL)) {
-                await this.runDuckDBQuery(statement);
+        // In non-shareable mode this whole method runs on every wake because
+        // the underlying DuckDBInstance was destroyed by `idle()`/`close()`,
+        // so the connection-level state (SETs, LOADs, user setupSQL effects)
+        // is gone and must be replayed.
+        //
+        // In shareable mode the `:memory:` primary survives idle and so does
+        // its connection-level state. Replaying the baseline + user setupSQL
+        // on a second wake would re-execute non-idempotent user statements
+        // (CREATE TABLE, INSERT, ...) and would also fail outright under
+        // `lockConfiguration=true` because subsequent `SET`s are rejected
+        // once configuration is locked. Gate everything except the ATTACH on
+        // `connectionSetupRan` so it only runs the first time.
+        const reattachOnly = this.normalized.effectiveShareable && this.connectionSetupRan;
+        if (!reattachOnly) {
+            await this.applyFinalBaseline();
+        }
+        await this.attachIfShareable();
+        if (!reattachOnly) {
+            if (this.normalized.setupSQL) {
+                for (const statement of splitSetupSQL(this.normalized.setupSQL)) {
+                    await this.runDuckDBQuery(statement);
+                }
             }
-        }
-        if (this.normalized.lockConfiguration) {
-            await this.runDuckDBQuery('SET lock_configuration=true');
+            if (this.normalized.lockConfiguration) {
+                await this.runDuckDBQuery('SET lock_configuration=true');
+            }
+            this.connectionSetupRan = true;
         }
     }
-    buildInstanceOptions() {
+    /**
+     * Shareable mode: ATTACH the real database file into the in-memory
+     * primary and `USE` it, so unqualified table refs resolve into the
+     * attached file. Idempotent — guarded by `this.attached`. Runs after
+     * `applyFinalBaseline()` so the corresponding `SET allowed_directories`
+     * (sandboxed mode) has already been applied.
+     */
+    async attachIfShareable() {
+        if (!this.normalized.effectiveShareable)
+            return;
+        if (this.attached)
+            return;
+        const readOnlyClause = this.normalized.readOnly ? ' (READ_ONLY)' : '';
+        await this.runDuckDBQuery(`ATTACH ${(0, duckdb_config_1.sqlStringLiteral)(this.normalized.databasePath)} AS ${SHAREABLE_ATTACH_ALIAS}${readOnlyClause}`);
+        await this.runDuckDBQuery(`USE ${SHAREABLE_ATTACH_ALIAS}.main`);
+        this.attached = true;
+    }
+    buildInstanceOptions(opts) {
+        const forShareablePrimary = (opts === null || opts === void 0 ? void 0 : opts.forShareablePrimary) === true;
         const options = {
             custom_user_agent: `Malloy/${package_json_1.default.version}`,
         };
         if (this.normalized.motherDuckToken !== undefined) {
             options['motherduck_token'] = this.normalized.motherDuckToken;
         }
-        if (this.normalized.readOnly) {
+        // In shareable mode the primary is :memory: and must stay writable —
+        // temp tables, manifestTemporaryTable, and search-index work all live
+        // there. The user's `readOnly` applies to the ATTACHed real file via
+        // `(READ_ONLY)` in attachIfShareable().
+        if (this.normalized.readOnly && !forShareablePrimary) {
             options['access_mode'] = 'READ_ONLY';
         }
         if (this.normalized.autoloadKnownExtensions !== undefined) {
@@ -276,16 +349,40 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         }
     }
     async close() {
-        this.detachInstance();
+        if (this.normalized.effectiveShareable) {
+            await this.detachShareableFile();
+            if (this.ownedInstance) {
+                try {
+                    this.ownedInstance.closeSync();
+                }
+                catch {
+                    // Best effort during shutdown.
+                }
+                this.ownedInstance = null;
+            }
+        }
+        else {
+            this.detachInstance();
+        }
         this.connection = null;
         this.isSetup = undefined;
         this.setupError = undefined;
+        this.attached = false;
         this.closed = true;
     }
     async idle() {
         // No-op for in-memory: closing the instance silently destroys state.
         if (this.normalized.databasePath === ':memory:')
             return;
+        if (this.normalized.effectiveShareable) {
+            // Shareable mode: keep the in-memory primary (and its temp tables,
+            // schema cache, etc.) alive; just DETACH the real file so the OS
+            // file lock is released. Next `setup()` re-runs `setupOnce()`, which
+            // re-ATTACHes lazily on first use.
+            await this.detachShareableFile();
+            this.isSetup = undefined;
+            return;
+        }
         this.detachInstance();
         this.connection = null;
         this.isSetup = undefined;
@@ -294,6 +391,23 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         // use and runs a fresh init() then. Doing it eagerly here would
         // re-acquire the lock immediately and defeat the point of idling.
     }
+    async detachShareableFile() {
+        if (!this.normalized.effectiveShareable)
+            return;
+        if (!this.attached || !this.connection)
+            return;
+        try {
+            await this.connection.run('USE memory.main');
+            await this.connection.run(`DETACH ${SHAREABLE_ATTACH_ALIAS}`);
+            this.attached = false;
+        }
+        catch {
+            // If DETACH fails the lock stays held; surfacing this error would
+            // mask the user's original op error. We leave `this.attached` as
+            // `true` so a later `attachIfShareable()` doesn't try to ATTACH on
+            // top of a still-attached alias.
+        }
+    }
     /**
      * Drop our entry from the shared `activeDBs` bookkeeping. If we were
      * the last sharer, close the underlying `DuckDBInstance`.

package/dist/native.js

@@ -149,6 +149,16 @@ const duckdb_connection_1 = require("./duckdb_connection");
             type: 'boolean',
             optional: true,
         },
+        {
+            name: 'shareable',
+            displayName: 'Shareable',
+            type: 'boolean',
+            optional: true,
+            description: 'When true, release the database file between operations so other ' +
+                'tools (malloy-cli, the duckdb CLI, another malloy host) can use ' +
+                'the same file while this connection is open. Adds a small ' +
+                'per-operation overhead. Default false.',
+        },
         {
             name: 'setupSQL',
             displayName: 'Setup SQL',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.390",
+  "version": "0.0.392",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -53,6 +53,7 @@
     "lint-fix": "eslint '**/*.ts{,x}' --fix",
     "test": "jest --config=../../jest.config.js",
     "build": "tsc --build",
+    "dev": "tsc --build",
     "clean": "tsc --build --clean && rm -f tsconfig.tsbuildinfo",
     "malloyc": "ts-node ../../scripts/malloy-to-json",
     "prepublishOnly": "npm run build"
@@ -60,7 +61,7 @@
   "dependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev13.0",
     "@duckdb/node-api": "1.4.4-r.1",
-    "@malloydata/malloy": "0.0.390",
+    "@malloydata/malloy": "0.0.392",
     "@motherduck/wasm-client": "^0.6.6",
     "apache-arrow": "^17.0.0",
     "web-worker": "^1.3.0"