@malloydata/db-duckdb 0.0.376 → 0.0.378

package/dist/duckdb_config.js

@@ -0,0 +1,375 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBConfigValidationError = void 0;
+exports.normalizeDuckDBConfig = normalizeDuckDBConfig;
+exports.buildDuckDBShareKey = buildDuckDBShareKey;
+exports.sqlStringLiteral = sqlStringLiteral;
+exports.sqlStringListLiteral = sqlStringListLiteral;
+exports.stringifyDuckDBOption = stringifyDuckDBOption;
+const path_1 = __importDefault(require("path"));
+const malloy_1 = require("@malloydata/malloy");
+const pathSecurity = __importStar(require("./path_security"));
+class DuckDBConfigValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'DuckDBConfigValidationError';
+    }
+}
+exports.DuckDBConfigValidationError = DuckDBConfigValidationError;
+const REQUIRED_BASELINE_EXTENSIONS = ['icu', 'json'];
+const DERIVED_TEMP_DIRECTORY_NAME = '.tmp';
+const DERIVED_SECRET_DIRECTORY_NAME = '.duckdb-secrets';
+function normalizeDuckDBConfig(config) {
+    var _a, _b, _c, _d;
+    const filesystemPolicy = parsePolicyValue(config['filesystemPolicy'], 'filesystemPolicy', ['open', 'sandboxed']);
+    const networkPolicy = parsePolicyValue(config['networkPolicy'], 'networkPolicy', ['open', 'closed']);
+    if (filesystemPolicy === 'sandboxed' && !pathSecurity.isPosixHost()) {
+        throw new DuckDBConfigValidationError('filesystemPolicy "sandboxed" is only supported on POSIX hosts');
+    }
+    const rawDatabasePath = (_b = (_a = readOptionalString(config, 'databasePath')) !== null && _a !== void 0 ? _a : readOptionalString(config, 'path')) !== null && _b !== void 0 ? _b : ':memory:';
+    const rawWorkingDirectory = readOptionalString(config, 'workingDirectory');
+    const rawSetupSQL = normalizeOptionalText(readOptionalString(config, 'setupSQL'));
+    const rawMotherDuckToken = normalizeOptionalText(readOptionalString(config, 'motherDuckToken'));
+    const readOnly = (_c = readOptionalBoolean(config, 'readOnly')) !== null && _c !== void 0 ? _c : false;
+    const additionalExtensions = normalizeExtensions(config['additionalExtensions'], 'additionalExtensions');
+    const enableExternalAccess = readOptionalBoolean(config, 'enableExternalAccess');
+    const lockConfiguration = readOptionalBoolean(config, 'lockConfiguration');
+    const autoloadKnownExtensions = readOptionalBoolean(config, 'autoloadKnownExtensions');
+    const autoinstallKnownExtensions = readOptionalBoolean(config, 'autoinstallKnownExtensions');
+    const allowCommunityExtensions = readOptionalBoolean(config, 'allowCommunityExtensions');
+    const allowUnsignedExtensions = readOptionalBoolean(config, 'allowUnsignedExtensions');
+    const tempFileEncryption = readOptionalBoolean(config, 'tempFileEncryption');
+    const threads = readOptionalInteger(config, 'threads');
+    const memoryLimit = readOptionalString(config, 'memoryLimit');
+    const rawTempDirectory = readOptionalString(config, 'tempDirectory');
+    const rawExtensionDirectory = readOptionalString(config, 'extensionDirectory');
+    const rawAllowedDirectories = readOptionalStringArray(config['allowedDirectories'], 'allowedDirectories');
+    const restricted = filesystemPolicy === 'sandboxed' || networkPolicy === 'closed';
+    const safetyPolicy = restricted
+        ? {
+            requiresPosixHost: filesystemPolicy === 'sandboxed',
+            requiresLockedConfiguration: true,
+            requiresNoSetupSQL: true,
+            requiresSandboxedPaths: filesystemPolicy === 'sandboxed',
+            requiresTempFileEncryption: true,
+            requiresSecretNeutralization: true,
+            requiredBaselineExtensions: REQUIRED_BASELINE_EXTENSIONS,
+            allowHttpfs: networkPolicy === 'open',
+            forbidAdditionalExtensions: true,
+            derivedTempDirectoryName: DERIVED_TEMP_DIRECTORY_NAME,
+        }
+        : undefined;
+    if ((safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.requiresNoSetupSQL) && rawSetupSQL !== undefined) {
+        throw new DuckDBConfigValidationError('setupSQL is not allowed when filesystemPolicy or networkPolicy requires a locked DuckDB baseline');
+    }
+    if ((safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.forbidAdditionalExtensions) &&
+        additionalExtensions.length > 0) {
+        throw new DuckDBConfigValidationError('additionalExtensions is not allowed when filesystemPolicy or networkPolicy requires a locked DuckDB baseline');
+    }
+    if (restricted && lockConfiguration === false) {
+        throw new DuckDBConfigValidationError('lockConfiguration cannot be false when filesystemPolicy or networkPolicy requires a locked DuckDB baseline');
+    }
+    if (restricted && tempFileEncryption === false) {
+        throw new DuckDBConfigValidationError('tempFileEncryption cannot be false when filesystemPolicy or networkPolicy requires a locked DuckDB baseline');
+    }
+    if (networkPolicy === 'closed') {
+        rejectConflictingBoolean(enableExternalAccess, 'enableExternalAccess', true, 'networkPolicy "closed"');
+        rejectConflictingBoolean(autoloadKnownExtensions, 'autoloadKnownExtensions', true, 'networkPolicy "closed"');
+        rejectConflictingBoolean(autoinstallKnownExtensions, 'autoinstallKnownExtensions', true, 'networkPolicy "closed"');
+        rejectConflictingBoolean(allowCommunityExtensions, 'allowCommunityExtensions', true, 'networkPolicy "closed"');
+        rejectConflictingBoolean(allowUnsignedExtensions, 'allowUnsignedExtensions', true, 'networkPolicy "closed"');
+        if (rawMotherDuckToken !== undefined) {
+            throw new DuckDBConfigValidationError('motherDuckToken is not allowed when networkPolicy is "closed"');
+        }
+    }
+    let workingDirectory;
+    if (rawWorkingDirectory !== undefined) {
+        workingDirectory = canonicalizeConfigPath(rawWorkingDirectory, 'workingDirectory', {
+            mustExist: filesystemPolicy === 'sandboxed',
+        });
+    }
+    const databasePath = canonicalizeDatabasePath(rawDatabasePath);
+    const isMotherDuck = isMotherDuckPath(databasePath);
+    if (networkPolicy === 'closed' &&
+        !isAllowedClosedNetworkDatabasePath(databasePath)) {
+        throw new DuckDBConfigValidationError(`databasePath "${rawDatabasePath}" is not allowed when networkPolicy is "closed"`);
+    }
+    if (networkPolicy === 'closed' && isMotherDuck) {
+        throw new DuckDBConfigValidationError('MotherDuck database paths are not allowed when networkPolicy is "closed"');
+    }
+    const normalizedAllowedDirectories = rawAllowedDirectories === undefined
+        ? undefined
+        : canonicalizeConfigPathList(rawAllowedDirectories, 'allowedDirectories');
+    let allowedDirectories = normalizedAllowedDirectories;
+    if (filesystemPolicy === 'sandboxed') {
+        if (allowedDirectories === undefined) {
+            if (workingDirectory === undefined) {
+                throw new DuckDBConfigValidationError('filesystemPolicy "sandboxed" requires either allowedDirectories or workingDirectory. If you expected workingDirectory to come from config.rootDirectory, verify that overlay is available.');
+            }
+            allowedDirectories = [workingDirectory];
+        }
+        if (workingDirectory !== undefined &&
+            !allowedDirectories.some(allowed => pathSecurity.isContainedPath(allowed, workingDirectory))) {
+            throw new DuckDBConfigValidationError('workingDirectory must be contained within allowedDirectories when filesystemPolicy is "sandboxed"');
+        }
+    }
+    let tempDirectory;
+    if (rawTempDirectory !== undefined) {
+        tempDirectory = canonicalizeConfigPath(rawTempDirectory, 'tempDirectory');
+    }
+    else if (filesystemPolicy === 'sandboxed') {
+        if (workingDirectory === undefined) {
+            throw new DuckDBConfigValidationError('filesystemPolicy "sandboxed" requires tempDirectory or workingDirectory so Malloy can derive a safe temp directory');
+        }
+        tempDirectory = path_1.default.join(workingDirectory, DERIVED_TEMP_DIRECTORY_NAME);
+    }
+    if (filesystemPolicy === 'sandboxed') {
+        if (allowedDirectories === undefined) {
+            throw new DuckDBConfigValidationError('filesystemPolicy "sandboxed" requires allowedDirectories');
+        }
+        if (tempDirectory === undefined) {
+            throw new DuckDBConfigValidationError('filesystemPolicy "sandboxed" requires tempDirectory');
+        }
+        if (!allowedDirectories.some(allowed => pathSecurity.isContainedPath(allowed, tempDirectory))) {
+            throw new DuckDBConfigValidationError('tempDirectory must be contained within allowedDirectories when filesystemPolicy is "sandboxed"');
+        }
+    }
+    const extensionDirectory = rawExtensionDirectory === undefined
+        ? undefined
+        : canonicalizeConfigPath(rawExtensionDirectory, 'extensionDirectory');
+    const secretDirectory = deriveRestrictedSecretDirectory({
+        requiresSecretNeutralization: (_d = safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.requiresSecretNeutralization) !== null && _d !== void 0 ? _d : false,
+        tempDirectory,
+        workingDirectory,
+    });
+    return {
+        name: config.name,
+        databasePath,
+        readOnly: databasePath === ':memory:' ? false : readOnly,
+        workingDirectory,
+        filesystemPolicy,
+        networkPolicy,
+        safetyPolicy,
+        allowedDirectories,
+        enableExternalAccess: networkPolicy === 'closed' ? false : enableExternalAccess,
+        lockConfiguration: (safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.requiresLockedConfiguration)
+            ? true
+            : lockConfiguration,
+        autoloadKnownExtensions: networkPolicy === 'closed' ? false : autoloadKnownExtensions,
+        autoinstallKnownExtensions: networkPolicy === 'closed' ? false : autoinstallKnownExtensions,
+        allowCommunityExtensions: networkPolicy === 'closed' ? false : allowCommunityExtensions,
+        allowUnsignedExtensions: networkPolicy === 'closed' ? false : allowUnsignedExtensions,
+        tempFileEncryption: (safetyPolicy === null || safetyPolicy === void 0 ? void 0 : safetyPolicy.requiresTempFileEncryption)
+            ? true
+            : tempFileEncryption,
+        threads,
+        memoryLimit,
+        tempDirectory,
+        secretDirectory,
+        extensionDirectory,
+        motherDuckToken: rawMotherDuckToken,
+        additionalExtensions,
+        setupSQL: rawSetupSQL,
+    };
+}
+function buildDuckDBShareKey(config) {
+    var _a, _b, _c, _d, _e, _f, _g;
+    // secretDirectory is derived from policy + workingDirectory/tempDirectory,
+    // which already participate in the share key.
+    return (0, malloy_1.makeDigest)('duckdb-share-key-v1', config.databasePath, String(config.readOnly), config.filesystemPolicy, config.networkPolicy, (_a = config.setupSQL) !== null && _a !== void 0 ? _a : '', ...((_b = config.allowedDirectories) !== null && _b !== void 0 ? _b : []), config.enableExternalAccess === undefined
+        ? ''
+        : String(config.enableExternalAccess), config.lockConfiguration === undefined
+        ? ''
+        : String(config.lockConfiguration), config.autoloadKnownExtensions === undefined
+        ? ''
+        : String(config.autoloadKnownExtensions), config.autoinstallKnownExtensions === undefined
+        ? ''
+        : String(config.autoinstallKnownExtensions), config.allowCommunityExtensions === undefined
+        ? ''
+        : String(config.allowCommunityExtensions), config.allowUnsignedExtensions === undefined
+        ? ''
+        : String(config.allowUnsignedExtensions), config.tempFileEncryption === undefined
+        ? ''
+        : String(config.tempFileEncryption), config.threads === undefined ? '' : String(config.threads), (_c = config.memoryLimit) !== null && _c !== void 0 ? _c : '', (_d = config.tempDirectory) !== null && _d !== void 0 ? _d : '', (_e = config.workingDirectory) !== null && _e !== void 0 ? _e : '', ...[...config.additionalExtensions].sort(), (_f = config.extensionDirectory) !== null && _f !== void 0 ? _f : '', (_g = config.motherDuckToken) !== null && _g !== void 0 ? _g : '');
+}
+function sqlStringLiteral(value) {
+    return `'${value.replace(/'/g, "''")}'`;
+}
+function sqlStringListLiteral(values) {
+    return `[${values.map(value => sqlStringLiteral(value)).join(',')}]`;
+}
+function stringifyDuckDBOption(value) {
+    return String(value);
+}
+function parsePolicyValue(rawValue, fieldName, allowedValues) {
+    if (rawValue === undefined) {
+        return 'open';
+    }
+    if (typeof rawValue !== 'string') {
+        throw new DuckDBConfigValidationError(`${fieldName} must be one of ${allowedValues
+            .map(v => `"${v}"`)
+            .join(' or ')}, got ${typeof rawValue}`);
+    }
+    if (allowedValues.includes(rawValue)) {
+        return rawValue;
+    }
+    throw new DuckDBConfigValidationError(`${fieldName} must be one of ${allowedValues
+        .map(v => `"${v}"`)
+        .join(' or ')}, got "${rawValue}"`);
+}
+function readOptionalString(config, fieldName) {
+    const rawValue = config[fieldName];
+    if (rawValue === undefined) {
+        return undefined;
+    }
+    if (typeof rawValue !== 'string') {
+        throw new DuckDBConfigValidationError(`${fieldName} must be a string, got ${typeof rawValue}`);
+    }
+    return rawValue;
+}
+function readOptionalBoolean(config, fieldName) {
+    const rawValue = config[fieldName];
+    if (rawValue === undefined) {
+        return undefined;
+    }
+    if (typeof rawValue !== 'boolean') {
+        throw new DuckDBConfigValidationError(`${fieldName} must be a boolean, got ${typeof rawValue}`);
+    }
+    return rawValue;
+}
+function readOptionalInteger(config, fieldName) {
+    const rawValue = config[fieldName];
+    if (rawValue === undefined) {
+        return undefined;
+    }
+    if (typeof rawValue !== 'number' || !Number.isInteger(rawValue)) {
+        throw new DuckDBConfigValidationError(`${fieldName} must be an integer number`);
+    }
+    return rawValue;
+}
+function readOptionalStringArray(rawValue, fieldName) {
+    if (rawValue === undefined) {
+        return undefined;
+    }
+    if (!Array.isArray(rawValue)) {
+        throw new DuckDBConfigValidationError(`${fieldName} must be a JSON array of strings`);
+    }
+    if (!rawValue.every(value => typeof value === 'string')) {
+        throw new DuckDBConfigValidationError(`${fieldName} must be a JSON array of strings`);
+    }
+    return rawValue;
+}
+function normalizeExtensions(rawValue, fieldName) {
+    if (rawValue === undefined) {
+        return [];
+    }
+    const parts = (() => {
+        if (typeof rawValue === 'string') {
+            return rawValue.split(',');
+        }
+        if (Array.isArray(rawValue) &&
+            rawValue.every(value => typeof value === 'string')) {
+            return rawValue;
+        }
+        throw new DuckDBConfigValidationError(`${fieldName} must be a comma-separated string or an array of strings`);
+    })();
+    const normalized = parts
+        .map(value => value.trim())
+        .filter(value => value !== '');
+    return Array.from(new Set(normalized));
+}
+function normalizeOptionalText(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    const trimmed = value.trim();
+    return trimmed === '' ? undefined : value;
+}
+function deriveRestrictedSecretDirectory({ requiresSecretNeutralization, tempDirectory, workingDirectory, }) {
+    if (!requiresSecretNeutralization) {
+        return undefined;
+    }
+    if (tempDirectory !== undefined) {
+        return path_1.default.join(tempDirectory, DERIVED_SECRET_DIRECTORY_NAME);
+    }
+    if (workingDirectory !== undefined) {
+        return path_1.default.join(workingDirectory, DERIVED_SECRET_DIRECTORY_NAME);
+    }
+    throw new DuckDBConfigValidationError('restricted DuckDB policies require workingDirectory or tempDirectory so Malloy can isolate persistent DuckDB secrets');
+}
+function canonicalizeDatabasePath(databasePath) {
+    if (databasePath === ':memory:' || isLikelyRemoteDatabasePath(databasePath)) {
+        return databasePath;
+    }
+    return canonicalizeConfigPath(databasePath, 'databasePath');
+}
+function canonicalizeConfigPath(input, fieldName, options = {}) {
+    try {
+        return pathSecurity.canonicalizePath(input, options);
+    }
+    catch (error) {
+        throw new DuckDBConfigValidationError(`${fieldName} is invalid: ${errorMessage(error)}`);
+    }
+}
+function canonicalizeConfigPathList(paths, fieldName) {
+    return Array.from(new Set(paths.map(p => canonicalizeConfigPath(p, fieldName)).sort()));
+}
+function isAllowedClosedNetworkDatabasePath(databasePath) {
+    return (databasePath === ':memory:' || !isLikelyRemoteDatabasePath(databasePath));
+}
+function isLikelyRemoteDatabasePath(databasePath) {
+    return (isMotherDuckPath(databasePath) ||
+        /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(databasePath));
+}
+function isMotherDuckPath(databasePath) {
+    return (databasePath.startsWith('md:') || databasePath.startsWith('motherduck:'));
+}
+function rejectConflictingBoolean(actualValue, fieldName, forbiddenValue, reason) {
+    if (actualValue === forbiddenValue) {
+        throw new DuckDBConfigValidationError(`${fieldName} cannot be ${forbiddenValue} when ${reason} is enabled`);
+    }
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=duckdb_config.js.map

package/dist/duckdb_connection.d.ts

@@ -3,12 +3,26 @@ import { DuckDBInstance } from '@duckdb/node-api';
 import type { DuckDBConnection as DuckDBNodeConnection } from '@duckdb/node-api';
 import type { ConnectionConfig, QueryRecord, QueryOptionsReader, RunSQLOptions } from '@malloydata/malloy';
 export interface DuckDBConnectionOptions extends ConnectionConfig {
-    additionalExtensions?: string[];
+    additionalExtensions?: string[] | string;
     databasePath?: string;
     motherDuckToken?: string;
     workingDirectory?: string;
     readOnly?: boolean;
     setupSQL?: string;
+    filesystemPolicy?: 'open' | 'sandboxed';
+    networkPolicy?: 'open' | 'closed';
+    allowedDirectories?: string[];
+    enableExternalAccess?: boolean;
+    lockConfiguration?: boolean;
+    autoloadKnownExtensions?: boolean;
+    autoinstallKnownExtensions?: boolean;
+    allowCommunityExtensions?: boolean;
+    allowUnsignedExtensions?: boolean;
+    tempFileEncryption?: boolean;
+    threads?: number;
+    memoryLimit?: string;
+    tempDirectory?: string;
+    extensionDirectory?: string;
 }
 interface ActiveDB {
     instance: DuckDBInstance;
@@ -16,10 +30,11 @@ interface ActiveDB {
 }
 export declare class DuckDBConnection extends DuckDBCommon {
     readonly name: string;
-    private additionalExtensions;
-    private databasePath;
-    private workingDirectory;
-    private readOnly;
+    private readonly normalized;
+    private readonly shareKey;
+    private readonly digestDatabasePath;
+    private readonly digestWorkingDirectory?;
+    private readonly digestSetupSQL?;
     connecting: Promise<void>;
     protected connection: DuckDBNodeConnection | null;
     protected setupError: Error | undefined;
@@ -29,8 +44,14 @@ export declare class DuckDBConnection extends DuckDBCommon {
     constructor(name: string, databasePath?: string, workingDirectory?: string, queryOptions?: QueryOptionsReader);
     getDigest(): string;
     private init;
-    loadExtension(ext: string): Promise<void>;
     protected setup(): Promise<void>;
+    private setupOnce;
+    private buildInstanceOptions;
+    private shouldApplyEnableExternalAccessAtOpenTime;
+    private applyFinalBaseline;
+    private loadBaselineExtensions;
+    private shouldLoadHttpfs;
+    private loadExtension;
     protected runDuckDBQuery(sql: string): Promise<{
         rows: QueryRecord[];
         totalRows: number;