@malloydata/db-duckdb 0.0.195-dev241003204905 → 0.0.195

package/dist/duckdb.spec.js

@@ -24,6 +24,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const duckdb_common_1 = require("./duckdb_common");
 const duckdb_connection_1 = require("./duckdb_connection");
+const malloy_1 = require("@malloydata/malloy");
 const test_1 = require("@malloydata/malloy/test");
 const [describe] = (0, test_1.describeIfDatabaseAvailable)(['duckdb']);
 /*
@@ -69,17 +70,17 @@ describe('DuckDBConnection', () => {
             expect(runRawSQL).toHaveBeenCalledTimes(2);
         });
         it('caches sql schema', async () => {
-            await connection.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+            await connection.fetchSchemaForSQLStruct(SQL_BLOCK_1, {});
             expect(runRawSQL).toHaveBeenCalledTimes(1);
             await new Promise(resolve => setTimeout(resolve));
-            await connection.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+            await connection.fetchSchemaForSQLStruct(SQL_BLOCK_1, {});
             expect(runRawSQL).toHaveBeenCalledTimes(1);
         });
         it('refreshes sql schema', async () => {
-            await connection.fetchSchemaForSQLBlock(SQL_BLOCK_2, {});
+            await connection.fetchSchemaForSQLStruct(SQL_BLOCK_2, {});
             expect(runRawSQL).toHaveBeenCalledTimes(1);
             await new Promise(resolve => setTimeout(resolve));
-            await connection.fetchSchemaForSQLBlock(SQL_BLOCK_2, {
+            await connection.fetchSchemaForSQLStruct(SQL_BLOCK_2, {
                 refreshTimestamp: Date.now() + 10,
             });
             expect(runRawSQL).toHaveBeenCalledTimes(2);
@@ -106,24 +107,12 @@ describe('DuckDBConnection', () => {
             const structDef = makeStructDef();
             connection.fillStructDefFromTypeMap(structDef, { test: ARRAY_SCHEMA });
             expect(structDef.fields[0]).toEqual({
-                'name': 'test',
-                'type': 'struct',
-                'dialect': 'duckdb',
-                'structRelationship': {
-                    'fieldName': 'test',
-                    'isArray': true,
-                    'type': 'nested',
-                },
-                'structSource': {
-                    'type': 'nested',
-                },
-                'fields': [
-                    {
-                        'name': 'value',
-                        'type': 'number',
-                        'numberType': 'integer',
-                    },
-                ],
+                name: 'test',
+                type: 'array',
+                elementTypeDef: intTyp,
+                join: 'many',
+                dialect: 'duckdb',
+                fields: (0, malloy_1.arrayEachFields)({ type: 'number', numberType: 'integer' }),
             });
         });
         it('parses inline', () => {
@@ -131,29 +120,13 @@ describe('DuckDBConnection', () => {
             connection.fillStructDefFromTypeMap(structDef, { test: INLINE_SCHEMA });
             expect(structDef.fields[0]).toEqual({
                 'name': 'test',
-                'type': 'struct',
+                'type': 'record',
                 'dialect': 'duckdb',
-                'structRelationship': {
-                    'type': 'inline',
-                },
-                'structSource': {
-                    'type': 'inline',
-                },
+                'join': 'one',
                 'fields': [
-                    {
-                        'name': 'a',
-                        'type': 'number',
-                        'numberType': 'float',
-                    },
-                    {
-                        'name': 'b',
-                        'type': 'number',
-                        'numberType': 'integer',
-                    },
-                    {
-                        'name': 'c',
-                        'type': 'string',
-                    },
+                    { 'name': 'a', ...dblType },
+                    { 'name': 'b', ...intTyp },
+                    { 'name': 'c', ...strTyp },
                 ],
             });
         });
@@ -162,14 +135,10 @@ describe('DuckDBConnection', () => {
             connection.fillStructDefFromTypeMap(structDef, { test: NESTED_SCHEMA });
             expect(structDef.fields[0]).toEqual({
                 'name': 'test',
-                'type': 'struct',
+                'type': 'array',
+                'elementTypeDef': { type: 'record_element' },
                 'dialect': 'duckdb',
-                'structRelationship': {
-                    'fieldName': 'test',
-                    'isArray': false,
-                    'type': 'nested',
-                },
-                'structSource': { 'type': 'nested' },
+                'join': 'many',
                 'fields': [
                     { 'name': 'a', 'numberType': 'float', 'type': 'number' },
                     { 'name': 'b', 'numberType': 'integer', 'type': 'number' },
@@ -195,14 +164,11 @@ describe('DuckDBConnection', () => {
  */
 const makeStructDef = () => {
     return {
-        type: 'struct',
+        type: 'table',
         name: 'test',
         dialect: 'duckdb',
-        structSource: { type: 'table', tablePath: 'test' },
-        structRelationship: {
-            type: 'basetable',
-            connectionName: 'duckdb',
-        },
+        tablePath: 'test',
+        connection: 'duckdb',
         fields: [],
     };
 };
@@ -212,8 +178,11 @@ const makeStructDef = () => {
 //
 // Uses string value for table
 const SQL_BLOCK_1 = {
-    type: 'sqlBlock',
+    type: 'sql_select',
     name: 'block1',
+    dialect: 'duckdb',
+    connection: 'duckdb',
+    fields: [],
     selectStr: `
 SELECT
 created_at,
@@ -230,8 +199,11 @@ FROM "inventory_items.parquet"
 };
 // Uses read_parquet() for table
 const SQL_BLOCK_2 = {
-    type: 'sqlBlock',
+    type: 'sql_select',
     name: 'block2',
+    dialect: 'duckdb',
+    connection: 'duckdb',
+    fields: [],
     selectStr: `
 SELECT
 created_at,
@@ -255,4 +227,7 @@ const ARRAY_SCHEMA = 'integer[]';
 const INLINE_SCHEMA = 'STRUCT(a double, b integer, c varchar(60))';
 // STRUCT(....)[] is nested
 const NESTED_SCHEMA = 'STRUCT(a double, b integer, c varchar(60))[]';
+const intTyp = { type: 'number', numberType: 'integer' };
+const strTyp = { type: 'string' };
+const dblType = { type: 'number', numberType: 'float' };
 //# sourceMappingURL=duckdb.spec.js.map

package/dist/duckdb_common.d.ts

@@ -1,4 +1,4 @@
-import { FetchSchemaOptions, MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, QueryOptionsReader, QueryRunStats, RunSQLOptions, SQLBlock, StreamingConnection, StructDef, TestableConnection } from '@malloydata/malloy';
+import { MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, QueryOptionsReader, QueryRunStats, RunSQLOptions, StreamingConnection, StructDef, TestableConnection, SQLSourceDef, TableSourceDef } from '@malloydata/malloy';
 import { BaseConnection } from '@malloydata/malloy/connection';
 export interface DuckDBQueryOptions {
     rowLimit: number;
@@ -9,8 +9,6 @@ export declare abstract class DuckDBCommon extends BaseConnection implements Tes
     protected motherDuckToken: string | undefined;
     private readonly dialect;
     static DEFAULT_QUERY_OPTIONS: DuckDBQueryOptions;
-    private schemaCache;
-    private sqlSchemaCache;
     readonly name: string;
     get dialectName(): string;
     protected readQueryOptions(): DuckDBQueryOptions;
@@ -28,7 +26,7 @@ export declare abstract class DuckDBCommon extends BaseConnection implements Tes
     }>;
     runSQL(sql: string, options?: RunSQLOptions): Promise<MalloyQueryData>;
     abstract runSQLStream(sql: string, options: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
-    private getSQLBlockSchema;
+    fetchSelectSchema(sqlRef: SQLSourceDef): Promise<SQLSourceDef | string>;
     estimateQueryCost(_: string): Promise<QueryRunStats>;
     /**
      * Split's a structs columns declaration into individual columns
@@ -46,18 +44,7 @@ export declare abstract class DuckDBCommon extends BaseConnection implements Tes
         [name: string]: string;
     }): void;
     private schemaFromQuery;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
-        structDef: StructDef;
-        error?: undefined;
-    } | {
-        error: string;
-        structDef?: undefined;
-    }>;
-    fetchSchemaForTables(tables: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
-        schemas: Record<string, StructDef>;
-        errors: Record<string, string>;
-    }>;
-    private getTableSchema;
+    fetchTableSchema(tableKey: string, tablePath: string): Promise<TableSourceDef>;
     canStream(): this is StreamingConnection;
     test(): Promise<void>;
     abstract createHash(sqlCommand: string): Promise<string>;

package/dist/duckdb_common.js

@@ -34,7 +34,7 @@ const unquoteName = (name) => {
 };
 class DuckDBCommon extends connection_1.BaseConnection {
     get dialectName() {
-        return 'duckdb';
+        return this.dialect.name;
     }
     readQueryOptions() {
         const options = DuckDBCommon.DEFAULT_QUERY_OPTIONS;
@@ -55,8 +55,6 @@ class DuckDBCommon extends connection_1.BaseConnection {
         this.queryOptions = queryOptions;
         this.isMotherDuck = false;
         this.dialect = new malloy_1.DuckDBDialect();
-        this.schemaCache = new Map();
-        this.sqlSchemaCache = new Map();
         this.name = 'duckdb_common';
     }
     isPool() {
@@ -85,24 +83,10 @@ class DuckDBCommon extends connection_1.BaseConnection {
         }
         return { rows: result, totalRows: result.length };
     }
-    async getSQLBlockSchema(sqlRef) {
-        const structDef = {
-            type: 'struct',
-            dialect: 'duckdb',
-            name: sqlRef.name,
-            structSource: {
-                type: 'sql',
-                method: 'subquery',
-                sqlBlock: sqlRef,
-            },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.name,
-            },
-            fields: [],
-        };
-        await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.selectStr})`, structDef);
-        return structDef;
+    async fetchSelectSchema(sqlRef) {
+        const sqlDef = { ...sqlRef };
+        await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.selectStr})`, sqlDef);
+        return sqlDef;
     }
     async estimateQueryCost(_) {
         return {};
@@ -168,8 +152,6 @@ class DuckDBCommon extends connection_1.BaseConnection {
             let duckDBType = typeMap[fieldName];
             // Remove quotes from field name
             const name = unquoteName(fieldName);
-            // Remove DECIMAL(x,y) precision to simplify lookup
-            duckDBType = duckDBType.replace(/^DECIMAL\(\d+,\d+\)/g, 'DECIMAL');
             let malloyType = this.dialect.sqlTypeToMalloyType(duckDBType);
             const arrayMatch = duckDBType.match(/(?<duckDBType>.*)\[\]$/);
             if (arrayMatch && arrayMatch.groups) {
@@ -178,22 +160,23 @@ class DuckDBCommon extends connection_1.BaseConnection {
             const structMatch = duckDBType.match(/^STRUCT\((?<fields>.*)\)$/);
             if (structMatch && structMatch.groups) {
                 const newTypeMap = this.stringToTypeMap(structMatch.groups['fields']);
-                const innerStructDef = {
-                    type: 'struct',
-                    name,
-                    dialect: this.dialectName,
-                    structSource: { type: arrayMatch ? 'nested' : 'inline' },
-                    structRelationship: arrayMatch
-                        ? {
-                            type: 'nested',
-                            fieldName: name,
-                            isArray: false,
-                        }
-                        : {
-                            type: 'inline',
-                        },
-                    fields: [],
-                };
+                let innerStructDef;
+                const structhead = { name, dialect: this.dialectName, fields: [] };
+                if (arrayMatch) {
+                    innerStructDef = {
+                        type: 'array',
+                        elementTypeDef: { type: 'record_element' },
+                        join: 'many',
+                        ...structhead,
+                    };
+                }
+                else {
+                    innerStructDef = {
+                        type: 'record',
+                        join: 'one',
+                        ...structhead,
+                    };
+                }
                 this.fillStructDefFromTypeMap(innerStructDef, newTypeMap);
                 structDef.fields.push(innerStructDef);
             }
@@ -201,30 +184,17 @@ class DuckDBCommon extends connection_1.BaseConnection {
                 if (arrayMatch) {
                     malloyType = this.dialect.sqlTypeToMalloyType(duckDBType);
                     const innerStructDef = {
-                        type: 'struct',
+                        type: 'array',
+                        elementTypeDef: malloyType,
                         name,
                         dialect: this.dialectName,
-                        structSource: { type: 'nested' },
-                        structRelationship: {
-                            type: 'nested',
-                            fieldName: name,
-                            isArray: true,
-                        },
-                        fields: [{ ...malloyType, name: 'value' }],
+                        join: 'many',
+                        fields: (0, malloy_1.arrayEachFields)(malloyType),
                     };
                     structDef.fields.push(innerStructDef);
                 }
                 else {
-                    if (malloyType) {
-                        structDef.fields.push({ ...malloyType, name });
-                    }
-                    else {
-                        structDef.fields.push({
-                            type: 'sql native',
-                            rawType: duckDBType.toLowerCase(),
-                            name,
-                        });
-                    }
+                    structDef.fields.push({ ...malloyType, name });
                 }
             }
         }
@@ -237,64 +207,13 @@ class DuckDBCommon extends connection_1.BaseConnection {
         }
         this.fillStructDefFromTypeMap(structDef, typeMap);
     }
-    async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
-        const key = sqlRef.name;
-        let inCache = this.sqlSchemaCache.get(key);
-        if (!inCache ||
-            (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
-            const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
-            try {
-                inCache = {
-                    structDef: await this.getSQLBlockSchema(sqlRef),
-                    timestamp,
-                };
-            }
-            catch (error) {
-                inCache = { error: error.message, timestamp };
-            }
-            this.sqlSchemaCache.set(key, inCache);
-        }
-        return inCache;
-    }
-    async fetchSchemaForTables(tables, { refreshTimestamp }) {
-        const schemas = {};
-        const errors = {};
-        for (const tableKey in tables) {
-            let inCache = this.schemaCache.get(tableKey);
-            if (!inCache ||
-                (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
-                const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
-                const tablePath = tables[tableKey];
-                try {
-                    inCache = {
-                        schema: await this.getTableSchema(tableKey, tablePath),
-                        timestamp,
-                    };
-                    this.schemaCache.set(tableKey, inCache);
-                }
-                catch (error) {
-                    inCache = { error: error.message, timestamp };
-                }
-            }
-            if (inCache.schema !== undefined) {
-                schemas[tableKey] = inCache.schema;
-            }
-            else {
-                errors[tableKey] = inCache.error || 'Unknown schema fetch error';
-            }
-        }
-        return { schemas, errors };
-    }
-    async getTableSchema(tableKey, tablePath) {
+    async fetchTableSchema(tableKey, tablePath) {
         const structDef = {
-            type: 'struct',
+            type: 'table',
             name: tableKey,
-            dialect: 'duckdb',
-            structSource: { type: 'table', tablePath },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.name,
-            },
+            dialect: this.dialectName,
+            tablePath,
+            connection: this.name,
             fields: [],
         };
         const quotedTablePath = tablePath.match(/[:*/]/)

package/dist/duckdb_wasm_connection.d.ts

@@ -1,5 +1,5 @@
 import * as duckdb from '@duckdb/duckdb-wasm';
-import { FetchSchemaOptions, QueryDataRow, QueryOptionsReader, RunSQLOptions, StructDef, SQLBlock, ConnectionConfig } from '@malloydata/malloy';
+import { FetchSchemaOptions, QueryDataRow, QueryOptionsReader, RunSQLOptions, SQLSourceDef, ConnectionConfig, TableSourceDef } from '@malloydata/malloy';
 import { StructRow } from 'apache-arrow';
 import { DuckDBCommon } from './duckdb_common';
 /**
@@ -53,15 +53,15 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     }>;
     runSQLStream(sql: string, { rowLimit, abortSignal }?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
     private findTables;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock, options: FetchSchemaOptions): Promise<{
-        structDef: StructDef;
+    fetchSchemaForSQLStruct(sqlRef: SQLSourceDef, options: FetchSchemaOptions): Promise<{
+        structDef: SQLSourceDef;
         error?: undefined;
     } | {
         error: string;
         structDef?: undefined;
     }>;
     fetchSchemaForTables(missing: Record<string, string>, options: FetchSchemaOptions): Promise<{
-        schemas: Record<string, StructDef>;
+        schemas: Record<string, TableSourceDef>;
         errors: Record<string, string>;
     }>;
     close(): Promise<void>;

package/dist/duckdb_wasm_connection.js

@@ -336,7 +336,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             await this.remoteFileStatus[tablePath];
         }
     }
-    async fetchSchemaForSQLBlock(sqlRef, options) {
+    async fetchSchemaForSQLStruct(sqlRef, options) {
         const tables = [];
         for (const match of sqlRef.selectStr.matchAll(TABLE_MATCH)) {
             tables.push(match[2] || match[3]);
@@ -345,7 +345,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             tables.push(match[2] || match[3]);
         }
         await this.findTables(tables, options);
-        return super.fetchSchemaForSQLBlock(sqlRef, options);
+        return super.fetchSchemaForSQLStruct(sqlRef, options);
     }
     async fetchSchemaForTables(missing, options) {
         const tables = Object.values(missing);

package/dist/duckdb_wasm_connection.spec.js

@@ -39,10 +39,8 @@ describe('DuckDBWasmConnection', () => {
     });
     beforeEach(() => {
         jest
-            .spyOn(duckdb_common_1.DuckDBCommon.prototype, 'fetchSchemaForSQLBlock')
-            .mockResolvedValue({
-            error: 'mocked',
-        });
+            .spyOn(duckdb_common_1.DuckDBCommon.prototype, 'fetchSelectSchema')
+            .mockResolvedValue('mocked');
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         findTables = jest.spyOn(connection, 'findTables');
     });
@@ -50,7 +48,7 @@ describe('DuckDBWasmConnection', () => {
         jest.resetAllMocks();
     });
     it('finds simple tables in SQL', async () => {
-        await connection.fetchSchemaForSQLBlock({
+        await connection.fetchSchemaForSQLStruct({
             selectStr: `
 SELECT
   created_at,
@@ -68,7 +66,7 @@ FROM "inventory_items.parquet"
         expect(findTables).toHaveBeenCalledWith(['order_items.parquet', 'inventory_items.parquet'], {});
     });
     it('finds table functions in SQL', async () => {
-        await connection.fetchSchemaForSQLBlock({
+        await connection.fetchSchemaForSQLStruct({
             selectStr: `
 SELECT
   created_at,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.195-dev241003204905",
+  "version": "0.0.195",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -41,7 +41,7 @@
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "1.28.1-dev242.0",
-    "@malloydata/malloy": "^0.0.195-dev241003204905",
+    "@malloydata/malloy": "^0.0.195",
     "@motherduck/wasm-client": "^0.6.3",
     "apache-arrow": "^16.0.0",
     "duckdb": "1.0.0",