@malloydata/db-duckdb 0.0.377 → 0.0.379

package/dist/duckdb_connection.js

@@ -30,160 +30,207 @@ const duckdb_common_1 = require("./duckdb_common");
 const node_api_1 = require("@duckdb/node-api");
 const malloy_1 = require("@malloydata/malloy");
 const package_json_1 = __importDefault(require("@malloydata/malloy/package.json"));
+const duckdb_config_1 = require("./duckdb_config");
 class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     constructor(arg, arg2, workingDirectory, queryOptions) {
+        var _a;
         super();
-        this.additionalExtensions = [];
-        this.databasePath = ':memory:';
-        this.workingDirectory = '.';
-        this.readOnly = false;
         this.connection = null;
+        const options = typeof arg === 'string'
+            ? buildLegacyOptions(arg, arg2, workingDirectory)
+            : arg;
+        this.name = options.name;
         if (typeof arg === 'string') {
-            this.name = arg;
-            if (typeof arg2 === 'string') {
-                this.databasePath = arg2;
-            }
-            if (typeof workingDirectory === 'string') {
-                this.workingDirectory = workingDirectory;
-            }
             if (queryOptions) {
                 this.queryOptions = queryOptions;
             }
         }
-        else {
-            this.name = arg.name;
-            if (arg2) {
-                this.queryOptions = arg2;
-            }
-            if (typeof arg.readOnly === 'boolean') {
-                this.readOnly = arg.readOnly;
-            }
-            if (typeof arg.databasePath === 'string') {
-                this.databasePath = arg.databasePath;
-            }
-            if (typeof arg.workingDirectory === 'string') {
-                this.workingDirectory = arg.workingDirectory;
-            }
-            if (typeof arg.motherDuckToken === 'string') {
-                this.motherDuckToken = arg.motherDuckToken;
-            }
-            if (Array.isArray(arg.additionalExtensions)) {
-                this.additionalExtensions = arg.additionalExtensions;
-            }
-            if (typeof arg.setupSQL === 'string') {
-                this.setupSQL = arg.setupSQL;
-            }
-        }
-        if (this.databasePath === ':memory:') {
-            this.readOnly = false;
+        else if (arg2) {
+            this.queryOptions = arg2;
         }
+        this.digestDatabasePath = (_a = options.databasePath) !== null && _a !== void 0 ? _a : ':memory:';
+        this.digestWorkingDirectory = options.workingDirectory;
+        this.digestSetupSQL =
+            typeof options.setupSQL === 'string' ? options.setupSQL : undefined;
+        this.normalized = (0, duckdb_config_1.normalizeDuckDBConfig)(options);
+        this.shareKey = (0, duckdb_config_1.buildDuckDBShareKey)(this.normalized);
         this.isMotherDuck =
-            this.databasePath.startsWith('md:') ||
-                this.databasePath.startsWith('motherduck:');
+            this.normalized.databasePath.startsWith('md:') ||
+                this.normalized.databasePath.startsWith('motherduck:');
+        this.motherDuckToken = this.normalized.motherDuckToken;
+        this.setupSQL = this.normalized.setupSQL;
         this.connecting = this.init();
     }
     getDigest() {
-        return (0, malloy_1.makeDigest)('duckdb', this.databasePath, this.workingDirectory, this.setupSQL);
+        return (0, malloy_1.makeDigest)('duckdb', this.digestDatabasePath, this.digestWorkingDirectory, this.digestSetupSQL);
     }
     async init() {
         try {
-            if (this.databasePath in DuckDBConnection.activeDBs) {
-                const activeDB = DuckDBConnection.activeDBs[this.databasePath];
-                this.connection = await activeDB.instance.connect();
-                activeDB.connections.push(this.connection);
-            }
-            else {
-                const config = {
-                    custom_user_agent: `Malloy/${package_json_1.default.version}`,
-                };
-                if (this.isMotherDuck) {
-                    if (!this.motherDuckToken &&
-                        !process.env['motherduck_token'] &&
-                        !process.env['MOTHERDUCK_TOKEN']) {
-                        this.setupError = new Error('Please set your MotherDuck Token');
-                        return;
-                    }
-                    if (this.motherDuckToken) {
-                        config['motherduck_token'] = this.motherDuckToken;
-                    }
-                }
-                if (this.readOnly) {
-                    config['access_mode'] = 'READ_ONLY';
-                }
-                const instance = await node_api_1.DuckDBInstance.create(this.databasePath, config);
-                this.connection = await instance.connect();
-                const activeDB = {
-                    instance,
-                    connections: [this.connection],
-                };
-                DuckDBConnection.activeDBs[this.databasePath] = activeDB;
+            const cached = DuckDBConnection.activeDBs[this.shareKey];
+            if (cached) {
+                this.connection = await cached.instance.connect();
+                cached.connections.push(this.connection);
+                return;
             }
+            const instance = await node_api_1.DuckDBInstance.create(this.normalized.databasePath, this.buildInstanceOptions());
+            this.connection = await instance.connect();
+            DuckDBConnection.activeDBs[this.shareKey] = {
+                instance,
+                connections: [this.connection],
+            };
         }
         catch (err) {
             this.setupError = err instanceof Error ? err : new Error(String(err));
         }
     }
-    async loadExtension(ext) {
-        try {
-            await this.runDuckDBQuery(`INSTALL '${ext}'`);
-            await this.runDuckDBQuery(`LOAD '${ext}'`);
-        }
-        catch (error) {
-            // eslint-disable-next-line no-console
-            console.error('Unable to load ${ext} extension', error);
-        }
-    }
     async setup() {
         if (this.setupError) {
             throw this.setupError;
         }
-        const doSetup = async () => {
-            if (this.workingDirectory) {
-                await this.runDuckDBQuery(`SET FILE_SEARCH_PATH='${this.workingDirectory}'`);
-            }
-            const extensions = [
-                'json',
-                'httpfs',
-                'icu',
-                ...this.additionalExtensions,
-            ];
-            if (this.databasePath.startsWith('md:')) {
-                extensions.push('motherduck');
-            }
-            for (const ext of extensions) {
-                await this.loadExtension(ext);
+        await this.connecting;
+        if (this.setupError) {
+            throw this.setupError;
+        }
+        if (!this.isSetup) {
+            this.isSetup = this.setupOnce();
+        }
+        await this.isSetup;
+    }
+    async setupOnce() {
+        await this.applyFinalBaseline();
+        if (this.normalized.setupSQL) {
+            for (const statement of splitSetupSQL(this.normalized.setupSQL)) {
+                await this.runDuckDBQuery(statement);
             }
-            const setupCmds = ["SET TimeZone='UTC'"];
-            for (const cmd of setupCmds) {
-                try {
-                    await this.runDuckDBQuery(cmd);
-                }
-                catch (error) {
-                    // eslint-disable-next-line no-console
-                    console.error(`duckdb setup ${cmd} => ${error}`);
+        }
+        if (this.normalized.lockConfiguration) {
+            await this.runDuckDBQuery('SET lock_configuration=true');
+        }
+    }
+    buildInstanceOptions() {
+        const options = {
+            custom_user_agent: `Malloy/${package_json_1.default.version}`,
+        };
+        if (this.normalized.motherDuckToken !== undefined) {
+            options['motherduck_token'] = this.normalized.motherDuckToken;
+        }
+        if (this.normalized.readOnly) {
+            options['access_mode'] = 'READ_ONLY';
+        }
+        if (this.normalized.autoloadKnownExtensions !== undefined) {
+            options['autoload_known_extensions'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.autoloadKnownExtensions);
+        }
+        if (this.normalized.autoinstallKnownExtensions !== undefined) {
+            options['autoinstall_known_extensions'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.autoinstallKnownExtensions);
+        }
+        if (this.normalized.allowCommunityExtensions !== undefined) {
+            options['allow_community_extensions'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.allowCommunityExtensions);
+        }
+        if (this.normalized.allowUnsignedExtensions !== undefined) {
+            options['allow_unsigned_extensions'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.allowUnsignedExtensions);
+        }
+        if (this.normalized.tempFileEncryption !== undefined) {
+            options['temp_file_encryption'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.tempFileEncryption);
+        }
+        if (this.normalized.threads !== undefined) {
+            options['threads'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.threads);
+        }
+        if (this.normalized.memoryLimit !== undefined) {
+            options['memory_limit'] = this.normalized.memoryLimit;
+        }
+        if (this.normalized.tempDirectory !== undefined) {
+            options['temp_directory'] = this.normalized.tempDirectory;
+        }
+        if (this.normalized.extensionDirectory !== undefined) {
+            options['extension_directory'] = this.normalized.extensionDirectory;
+        }
+        if (this.shouldApplyEnableExternalAccessAtOpenTime()) {
+            options['enable_external_access'] = (0, duckdb_config_1.stringifyDuckDBOption)(this.normalized.enableExternalAccess);
+        }
+        return options;
+    }
+    shouldApplyEnableExternalAccessAtOpenTime() {
+        // DuckDB's Node API does not currently accept allowed_directories as an
+        // open-time option, and DuckDB rejects SET allowed_directories after
+        // enable_external_access=false. Apply the disable at open time unless a
+        // post-connect allowlist SET is required.
+        return (this.normalized.enableExternalAccess !== undefined &&
+            this.normalized.allowedDirectories === undefined);
+    }
+    async applyFinalBaseline() {
+        if (this.normalized.allowedDirectories !== undefined) {
+            await this.runDuckDBQuery(`SET allowed_directories=${(0, duckdb_config_1.sqlStringListLiteral)(this.normalized.allowedDirectories)}`);
+        }
+        if (this.normalized.enableExternalAccess !== undefined &&
+            !this.shouldApplyEnableExternalAccessAtOpenTime()) {
+            await this.runDuckDBQuery(`SET enable_external_access=${this.normalized.enableExternalAccess}`);
+        }
+        if (this.normalized.workingDirectory !== undefined) {
+            await this.runDuckDBQuery(`SET FILE_SEARCH_PATH=${(0, duckdb_config_1.sqlStringLiteral)(this.normalized.workingDirectory)}`);
+        }
+        if (this.normalized.secretDirectory !== undefined) {
+            await this.runDuckDBQuery(`SET secret_directory=${(0, duckdb_config_1.sqlStringLiteral)(this.normalized.secretDirectory)}`);
+        }
+        await this.runDuckDBQuery("SET TimeZone='UTC'");
+        await this.loadBaselineExtensions();
+    }
+    async loadBaselineExtensions() {
+        if (this.normalized.networkPolicy === 'closed') {
+            await this.loadExtension('json', { allowInstall: false, required: true });
+            await this.loadExtension('icu', { allowInstall: false, required: true });
+            return;
+        }
+        const allowInstall = this.normalized.enableExternalAccess !== false;
+        await this.loadExtension('json', { allowInstall, required: false });
+        await this.loadExtension('icu', { allowInstall, required: false });
+        if (this.shouldLoadHttpfs()) {
+            await this.loadExtension('httpfs', { allowInstall, required: false });
+        }
+        for (const extension of this.normalized.additionalExtensions) {
+            await this.loadExtension(extension, { allowInstall, required: false });
+        }
+        if (this.isMotherDuck) {
+            await this.loadExtension('motherduck', { allowInstall, required: false });
+        }
+    }
+    shouldLoadHttpfs() {
+        if (this.normalized.networkPolicy === 'closed') {
+            return false;
+        }
+        return this.normalized.enableExternalAccess !== false;
+    }
+    async loadExtension(extension, options) {
+        try {
+            await this.runDuckDBQuery(`LOAD ${(0, duckdb_config_1.sqlStringLiteral)(extension)}`);
+            return;
+        }
+        catch (loadError) {
+            if (!options.allowInstall) {
+                if (options.required) {
+                    throw loadError;
                 }
+                // eslint-disable-next-line no-console
+                console.error(`Unable to load DuckDB extension "${extension}"`, loadError);
+                return;
             }
-            if (this.setupSQL) {
-                for (const stmt of this.setupSQL.split(';\n')) {
-                    const trimmed = stmt.trim();
-                    if (trimmed) {
-                        await this.runDuckDBQuery(trimmed);
-                    }
-                }
+        }
+        try {
+            await this.runDuckDBQuery(`INSTALL ${(0, duckdb_config_1.sqlStringLiteral)(extension)}`);
+            await this.runDuckDBQuery(`LOAD ${(0, duckdb_config_1.sqlStringLiteral)(extension)}`);
+        }
+        catch (error) {
+            if (options.required) {
+                throw error;
             }
-        };
-        await this.connecting;
-        if (!this.isSetup) {
-            this.isSetup = doSetup();
+            // eslint-disable-next-line no-console
+            console.error(`Unable to load DuckDB extension "${extension}"`, error);
         }
-        await this.isSetup;
     }
     async runDuckDBQuery(sql) {
         if (!this.connection) {
             throw new Error('Connection not open');
         }
         const result = await this.connection.run(sql);
-        // getRowObjectsJson() converts nested types (LIST, STRUCT) to JS arrays/objects
         const rows = (await result.getRowObjectsJson());
         return {
             rows,
@@ -216,12 +263,12 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         }
     }
     async close() {
-        const activeDB = DuckDBConnection.activeDBs[this.databasePath];
+        const activeDB = DuckDBConnection.activeDBs[this.shareKey];
         if (activeDB) {
             activeDB.connections = activeDB.connections.filter(connection => connection !== this.connection);
             if (activeDB.connections.length === 0) {
                 activeDB.instance.closeSync();
-                delete DuckDBConnection.activeDBs[this.databasePath];
+                delete DuckDBConnection.activeDBs[this.shareKey];
             }
         }
     }
@@ -230,9 +277,9 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
      * to release file locks between test runs.
      */
     static closeAllInstances() {
-        for (const path of Object.keys(DuckDBConnection.activeDBs)) {
+        for (const key of Object.keys(DuckDBConnection.activeDBs)) {
             try {
-                DuckDBConnection.activeDBs[path].instance.closeSync();
+                DuckDBConnection.activeDBs[key].instance.closeSync();
             }
             catch {
                 // Ignore errors during cleanup
@@ -243,4 +290,18 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
 }
 exports.DuckDBConnection = DuckDBConnection;
 DuckDBConnection.activeDBs = {};
+function buildLegacyOptions(name, databasePathOrQueryOptions, workingDirectory) {
+    const options = { name };
+    if (typeof databasePathOrQueryOptions === 'string') {
+        options.databasePath = databasePathOrQueryOptions;
+    }
+    options.workingDirectory = workingDirectory !== null && workingDirectory !== void 0 ? workingDirectory : '.';
+    return options;
+}
+function splitSetupSQL(setupSQL) {
+    return setupSQL
+        .split(';\n')
+        .map(statement => statement.trim())
+        .filter(statement => statement !== '');
+}
 //# sourceMappingURL=duckdb_connection.js.map

package/dist/native.js

@@ -15,13 +15,6 @@ const duckdb_connection_1 = require("./duckdb_connection");
             options['databasePath'] = options['path'];
             delete options['path'];
         }
-        // Parse comma-separated extensions string into array
-        if (typeof options['additionalExtensions'] === 'string') {
-            options['additionalExtensions'] = options['additionalExtensions']
-                .split(',')
-                .map(s => s.trim())
-                .filter(s => s.length > 0);
-        }
         return new duckdb_connection_1.DuckDBConnection(options);
     },
     properties: [
@@ -44,6 +37,92 @@ const duckdb_connection_1 = require("./duckdb_connection");
             // happens to live inside the project.
             default: { config: 'rootDirectory' },
         },
+        {
+            name: 'filesystemPolicy',
+            displayName: 'Filesystem Policy',
+            type: 'string',
+            optional: true,
+            requireLiteralString: true,
+        },
+        {
+            name: 'networkPolicy',
+            displayName: 'Network Policy',
+            type: 'string',
+            optional: true,
+            requireLiteralString: true,
+        },
+        {
+            name: 'allowedDirectories',
+            displayName: 'Allowed Directories',
+            type: 'json',
+            optional: true,
+        },
+        {
+            name: 'enableExternalAccess',
+            displayName: 'Enable External Access',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'lockConfiguration',
+            displayName: 'Lock Configuration',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'autoloadKnownExtensions',
+            displayName: 'Autoload Known Extensions',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'autoinstallKnownExtensions',
+            displayName: 'Autoinstall Known Extensions',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'allowCommunityExtensions',
+            displayName: 'Allow Community Extensions',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'allowUnsignedExtensions',
+            displayName: 'Allow Unsigned Extensions',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'tempFileEncryption',
+            displayName: 'Temp File Encryption',
+            type: 'boolean',
+            optional: true,
+        },
+        {
+            name: 'threads',
+            displayName: 'Threads',
+            type: 'number',
+            optional: true,
+        },
+        {
+            name: 'memoryLimit',
+            displayName: 'Memory Limit',
+            type: 'string',
+            optional: true,
+        },
+        {
+            name: 'tempDirectory',
+            displayName: 'Temp Directory',
+            type: 'file',
+            optional: true,
+        },
+        {
+            name: 'extensionDirectory',
+            displayName: 'Extension Directory',
+            type: 'file',
+            optional: true,
+        },
         {
             name: 'motherDuckToken',
             displayName: 'MotherDuck Token',

package/dist/path_security.d.ts

@@ -0,0 +1,7 @@
+export interface CanonicalPathOptions {
+    mustExist?: boolean;
+}
+export declare function isPosixHost(): boolean;
+export declare function canonicalizePath(input: string, { mustExist }?: CanonicalPathOptions): string;
+export declare function canonicalizePathList(paths: string[]): string[];
+export declare function isContainedPath(parent: string, child: string): boolean;

package/dist/path_security.js

@@ -0,0 +1,83 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPosixHost = isPosixHost;
+exports.canonicalizePath = canonicalizePath;
+exports.canonicalizePathList = canonicalizePathList;
+exports.isContainedPath = isContainedPath;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+function isPosixHost() {
+    return path_1.default.sep === '/';
+}
+function canonicalizePath(input, { mustExist = false } = {}) {
+    if (input.trim() === '') {
+        throw new Error('path must not be empty');
+    }
+    const resolved = path_1.default.resolve(input);
+    const canonical = (() => {
+        try {
+            return fs_1.default.realpathSync.native(resolved);
+        }
+        catch (error) {
+            if (mustExist || !isENOENT(error)) {
+                throw error;
+            }
+            return canonicalizeMissingPath(resolved);
+        }
+    })();
+    return stripTrailingSeparator(path_1.default.normalize(canonical));
+}
+function canonicalizePathList(paths) {
+    return Array.from(new Set(paths.map(p => canonicalizePath(p)).sort()));
+}
+function isContainedPath(parent, child) {
+    const relative = path_1.default.relative(parent, child);
+    return (relative === '' ||
+        (!relative.startsWith('..') && !path_1.default.isAbsolute(relative)));
+}
+function canonicalizeMissingPath(resolved) {
+    const parent = path_1.default.dirname(resolved);
+    if (parent === resolved) {
+        return resolved;
+    }
+    const realParent = (() => {
+        try {
+            return fs_1.default.realpathSync.native(parent);
+        }
+        catch (error) {
+            if (!isENOENT(error)) {
+                throw error;
+            }
+            return canonicalizeMissingPath(parent);
+        }
+    })();
+    return path_1.default.join(realParent, path_1.default.basename(resolved));
+}
+function isENOENT(error) {
+    return (typeof error === 'object' &&
+        error !== null &&
+        'code' in error &&
+        error.code === 'ENOENT');
+}
+function stripTrailingSeparator(value) {
+    const root = path_1.default.parse(value).root;
+    if (value === root) {
+        return value;
+    }
+    let end = value.length;
+    while (end > root.length && isPathSeparator(value.charCodeAt(end - 1))) {
+        end -= 1;
+    }
+    return end === value.length ? value : value.slice(0, end);
+}
+function isPathSeparator(charCode) {
+    return charCode === 47 || charCode === 92;
+}
+//# sourceMappingURL=path_security.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.377",
+  "version": "0.0.379",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -60,7 +60,7 @@
   "dependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev13.0",
     "@duckdb/node-api": "1.4.4-r.1",
-    "@malloydata/malloy": "0.0.377",
+    "@malloydata/malloy": "0.0.379",
     "@motherduck/wasm-client": "^0.6.6",
     "apache-arrow": "^17.0.0",
     "web-worker": "^1.3.0"