@malloydata/db-duckdb 0.0.130-dev240311192728 → 0.0.130-dev240312011825

package/dist/duckdb_common.d.ts

@@ -4,6 +4,8 @@ export interface DuckDBQueryOptions {
 }
 export declare abstract class DuckDBCommon implements TestableConnection, PersistSQLResults, StreamingConnection {
     protected queryOptions?: QueryOptionsReader | undefined;
+    protected isMotherDuck: boolean;
+    protected motherDuckToken: string | undefined;
     private readonly dialect;
     static DEFAULT_QUERY_OPTIONS: DuckDBQueryOptions;
     private schemaCache;

package/dist/duckdb_common.js

@@ -51,6 +51,7 @@ class DuckDBCommon {
     }
     constructor(queryOptions) {
         this.queryOptions = queryOptions;
+        this.isMotherDuck = false;
         this.dialect = new malloy_1.DuckDBDialect();
         this.schemaCache = new Map();
         this.sqlSchemaCache = new Map();

package/dist/duckdb_connection.d.ts

@@ -17,8 +17,6 @@ export declare class DuckDBConnection extends DuckDBCommon {
     private additionalExtensions;
     private databasePath;
     private workingDirectory;
-    private isMotherDuck;
-    private motherDuckToken;
     private readOnly;
     connecting: Promise<void>;
     protected connection: Connection | null;

package/dist/duckdb_connection.js

@@ -35,7 +35,6 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         this.additionalExtensions = [];
         this.databasePath = ':memory:';
         this.workingDirectory = '.';
-        this.isMotherDuck = false;
         this.readOnly = false;
         this.connection = null;
         if (typeof arg === 'string') {

package/dist/duckdb_wasm_connection.d.ts

@@ -1,13 +1,34 @@
-import * as duckdb from '@malloydata/duckdb-wasm';
+import * as duckdb from '@duckdb/duckdb-wasm';
 import { FetchSchemaOptions, QueryDataRow, QueryOptionsReader, RunSQLOptions, StructDef, SQLBlock, ConnectionConfig } from '@malloydata/malloy';
+import { StructRow } from 'apache-arrow';
 import { DuckDBCommon } from './duckdb_common';
+/**
+ * Arrow's toJSON() doesn't really do what I'd expect, since
+ * it still includes Arrow objects like DecimalBigNums and Vectors,
+ * so we need this fairly gross function to unwrap those.
+ *
+ * @param value Element from an Arrow StructRow.
+ * @return Vanilla Javascript value
+ */
+export declare const unwrapArrow: (value: unknown) => any;
+/**
+ * Process a single Arrow result row into a Malloy QueryDataRow
+ * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
+ * winds up converting DecimalBigNums to strings instead of numbers.
+ * For some reason a custom replacer only sees DecimalBigNums as
+ * strings, as well.
+ */
+export declare const unwrapRow: (row: StructRow) => QueryDataRow;
 type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export interface DuckDBWasmOptions extends ConnectionConfig {
+    additionalExtensions?: string[];
     databasePath?: string;
+    motherDuckToken: string | undefined;
     workingDirectory?: string;
 }
 export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly arg: string | DuckDBWasmOptions;
+    private additionalExtensions;
     readonly name: string;
     private databasePath;
     protected workingDirectory: string;
@@ -20,7 +41,7 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     private remoteFileStatus;
     constructor(options: DuckDBWasmOptions, queryOptions?: QueryOptionsReader);
     constructor(name: string, databasePath?: string | null, workingDirectory?: string, queryOptions?: QueryOptionsReader);
-    private init;
+    protected init(): Promise<void>;
     abstract getBundles(): duckdb.DuckDBBundles;
     get connection(): duckdb.AsyncDuckDBConnection | null;
     get database(): duckdb.AsyncDuckDB | null;

package/dist/duckdb_wasm_connection.js

@@ -48,13 +48,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DuckDBWASMConnection = void 0;
-const duckdb = __importStar(require("@malloydata/duckdb-wasm"));
+exports.DuckDBWASMConnection = exports.unwrapRow = exports.unwrapArrow = void 0;
+const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
 const web_worker_1 = __importDefault(require("web-worker"));
 const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
 const TABLE_MATCH = /FROM\s*('([^']*)'|"([^"]*)")/gi;
 const TABLE_FUNCTION_MATCH = /FROM\s+[a-z0-9_]+\(('([^']*)'|"([^"]*)")/gi;
+const FILE_EXTS = ['.csv', '.tsv', '.parquet'];
 /**
  * Arrow's toJSON() doesn't really do what I'd expect, since
  * it still includes Arrow objects like DecimalBigNums and Vectors,
@@ -69,7 +70,7 @@ const unwrapArrow = (value) => {
         return value;
     }
     else if (value instanceof apache_arrow_1.Vector) {
-        return [...value].map(unwrapArrow);
+        return [...value].map(exports.unwrapArrow);
     }
     else if (value instanceof Date) {
         return value;
@@ -88,14 +89,22 @@ const unwrapArrow = (value) => {
             return Number(obj[Symbol.toPrimitive]());
         }
         else if (Array.isArray(value)) {
-            return value.map(unwrapArrow);
+            return value.map(exports.unwrapArrow);
+        }
+        else if (obj['microseconds'] && obj['timezone'] === null) {
+            // Convert epoch µs to ms
+            return Number(obj['microseconds']) / 1000;
+        }
+        else if (obj['days']) {
+            // Convert epoch day to Date
+            return new Date(obj['days'] * 8.64e7);
         }
         else {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const result = {};
             for (const key in obj) {
                 if (Object.prototype.hasOwnProperty.call(obj, key)) {
-                    result[key] = unwrapArrow(obj[key]);
+                    result[key] = (0, exports.unwrapArrow)(obj[key]);
                 }
             }
             return result;
@@ -103,6 +112,7 @@ const unwrapArrow = (value) => {
     }
     return value;
 };
+exports.unwrapArrow = unwrapArrow;
 /**
  * Process a single Arrow result row into a Malloy QueryDataRow
  * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
@@ -111,19 +121,22 @@ const unwrapArrow = (value) => {
  * strings, as well.
  */
 const unwrapRow = (row) => {
-    return unwrapArrow(row.toJSON());
+    return (0, exports.unwrapArrow)(row.toJSON());
 };
+exports.unwrapRow = unwrapRow;
 /**
  * Process a duckedb Table into an array of Malloy QueryDataRows
  */
 const unwrapTable = (table) => {
-    return table.toArray().map(unwrapRow);
+    return table.toArray().map(exports.unwrapRow);
 };
 const isNode = () => typeof navigator === 'undefined';
 class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     constructor(arg, arg2, workingDirectory, queryOptions) {
+        var _a, _b;
         super();
         this.arg = arg;
+        this.additionalExtensions = [];
         this.databasePath = null;
         this.workingDirectory = '/';
         this._connection = null;
@@ -154,7 +167,17 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             if (typeof arg.workingDirectory === 'string') {
                 this.workingDirectory = arg.workingDirectory;
             }
+            if (typeof arg.motherDuckToken === 'string') {
+                this.motherDuckToken = arg.motherDuckToken;
+            }
+            if (Array.isArray(arg.additionalExtensions)) {
+                this.additionalExtensions = arg.additionalExtensions;
+            }
         }
+        this.isMotherDuck =
+            ((_a = this.databasePath) === null || _a === void 0 ? void 0 : _a.startsWith('md:')) ||
+                ((_b = this.databasePath) === null || _b === void 0 ? void 0 : _b.startsWith('motherduck:')) ||
+                false;
         this.connecting = this.init();
     }
     async init() {
@@ -204,10 +227,10 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             if (this.workingDirectory) {
                 await this.runDuckDBQuery(`SET FILE_SEARCH_PATH='${this.workingDirectory}'`);
             }
-            // Not quite ready for prime time
-            // for (const ext of ['json', 'httpfs', 'icu']) {
-            //   await this.loadExtension(ext);
-            // }
+            const extensions = ['json', 'icu', ...this.additionalExtensions];
+            for (const ext of extensions) {
+                await this.loadExtension(ext);
+            }
             const setupCmds = ["SET TimeZone='UTC'"];
             for (const cmd of setupCmds) {
                 try {
@@ -279,7 +302,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
                     (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
                     break;
                 }
-                yield unwrapRow(row);
+                yield (0, exports.unwrapRow)(row);
                 index++;
             }
         }
@@ -299,6 +322,11 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
         };
         await this.setup();
         for (const tablePath of tables) {
+            if (this.isMotherDuck &&
+                !tables.includes('/') &&
+                !FILE_EXTS.some(ext => tablePath.endsWith(ext))) {
+                continue;
+            }
             // http and s3 urls are handled by duckdb-wasm
             if (tablePath.match(/^https?:\/\//)) {
                 continue;

package/dist/duckdb_wasm_connection_browser.d.ts

@@ -1,6 +1,15 @@
-import * as duckdb from '@malloydata/duckdb-wasm';
+import * as duckdb from '@duckdb/duckdb-wasm';
 import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from './duckdb_wasm_connection';
+import { MDConnection } from '@motherduck/wasm-client';
+import { QueryDataRow } from '@malloydata/malloy';
 export declare class DuckDBWASMConnection extends DuckDBWASMConnectionBase {
+    protected _mdConnection: MDConnection | null;
     getBundles(): duckdb.DuckDBBundles;
+    init(): Promise<void>;
+    setup(): Promise<void>;
+    protected runDuckDBQuery(sql: string, abortSignal?: AbortSignal): Promise<{
+        rows: QueryDataRow[];
+        totalRows: number;
+    }>;
     createHash(sqlCommand: string): Promise<string>;
 }

package/dist/duckdb_wasm_connection_browser.js

@@ -46,12 +46,89 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDBWASMConnection = void 0;
-const duckdb = __importStar(require("@malloydata/duckdb-wasm"));
+const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
 const duckdb_wasm_connection_1 = require("./duckdb_wasm_connection");
+const wasm_client_1 = require("@motherduck/wasm-client");
 class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection {
+    constructor() {
+        super(...arguments);
+        this._mdConnection = null;
+    }
     getBundles() {
         return duckdb.getJsDelivrBundles();
     }
+    async init() {
+        if (this.isMotherDuck) {
+            if (!this.motherDuckToken) {
+                throw new Error('Please set your MotherDuck token');
+            }
+            const mdConnection = wasm_client_1.MDConnection.create({
+                mdToken: this.motherDuckToken,
+            });
+            await mdConnection.isInitialized();
+            this._mdConnection = mdConnection;
+            console.info('MotherDuck initialized');
+        }
+        else {
+            await super.init();
+        }
+    }
+    async setup() {
+        if (this.isMotherDuck) {
+            const doSetup = async () => {
+                const setupCmds = ["SET TimeZone='UTC'"];
+                for (const cmd of setupCmds) {
+                    try {
+                        await this.runDuckDBQuery(cmd);
+                    }
+                    catch (error) {
+                        // eslint-disable-next-line no-console
+                        console.error(`duckdb setup ${cmd} => ${error}`);
+                    }
+                }
+            };
+            await this.connecting;
+            if (!this.isSetup) {
+                this.isSetup = doSetup();
+            }
+            await this.isSetup;
+        }
+        else {
+            await super.setup();
+        }
+    }
+    async runDuckDBQuery(sql, abortSignal) {
+        if (this.isMotherDuck) {
+            if (this._mdConnection) {
+                const connection = this._mdConnection;
+                let queryId = undefined;
+                const cancel = () => {
+                    if (queryId) {
+                        connection.cancelQuery(queryId, 'Cancelled');
+                    }
+                };
+                abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.addEventListener('abort', cancel);
+                queryId = connection.enqueueQuery(sql);
+                if (queryId) {
+                    const result = await connection.evaluateQueuedQuery(queryId);
+                    if (result === null || result === void 0 ? void 0 : result.data) {
+                        const rows = (0, duckdb_wasm_connection_1.unwrapArrow)(result.data.toRows());
+                        const totalRows = result.data.rowCount;
+                        return {
+                            rows,
+                            totalRows,
+                        };
+                    }
+                    throw new Error('No data');
+                }
+                throw new Error('Failed to enqueue query');
+            }
+            throw new Error('MotherDuck not initialized');
+        }
+        else {
+            return super.runDuckDBQuery(sql, abortSignal);
+        }
+    }
     async createHash(sqlCommand) {
         const msgUint8 = new TextEncoder().encode(sqlCommand);
         const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8);

package/dist/duckdb_wasm_connection_node.d.ts

@@ -1,4 +1,4 @@
-import { DuckDBBundles } from '@malloydata/duckdb-wasm';
+import { DuckDBBundles } from '@duckdb/duckdb-wasm';
 import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from './duckdb_wasm_connection';
 export declare class DuckDBWASMConnection extends DuckDBWASMConnectionBase {
     getBundles(): DuckDBBundles;

package/dist/duckdb_wasm_connection_node.js

@@ -30,13 +30,13 @@ const crypto_1 = __importDefault(require("crypto"));
 const duckdb_wasm_connection_1 = require("./duckdb_wasm_connection");
 class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection {
     getBundles() {
-        const resolvePath = require.resolve('@malloydata/duckdb-wasm');
+        const resolvePath = require.resolve('@duckdb/duckdb-wasm');
         if (!resolvePath) {
-            throw new Error('Unable to resolve @malloydata/duckdb-wasm path');
+            throw new Error('Unable to resolve @duckdb/duckdb-wasm path');
         }
         const distMatch = resolvePath.match(/^.*\/dist\//);
         if (!distMatch) {
-            throw new Error('Unable to resolve @malloydata/duckdb-wasm dist path');
+            throw new Error('Unable to resolve @duckdb/duckdb-wasm dist path');
         }
         const dist = distMatch[0];
         return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.130-dev240311192728",
+  "version": "0.0.130-dev240312011825",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -40,9 +40,10 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@malloydata/duckdb-wasm": "0.0.6",
-    "@malloydata/malloy": "^0.0.130-dev240311192728",
-    "apache-arrow": "^13.0.0",
+    "@duckdb/duckdb-wasm": "1.28.1-dev106.0",
+    "@malloydata/malloy": "^0.0.130-dev240312011825",
+    "@motherduck/wasm-client": "^0.4.0",
+    "apache-arrow": "^14.0.0",
     "duckdb": "0.9.2",
     "web-worker": "^1.2.0"
   }