kepler.gl 3.1.0 → 3.1.1

package/src/duckdb/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@kepler.gl/duckdb",
   "author": "Shan He <heshan0131@gmail.com>",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "DuckDB plugin for Kepler.gl",
   "license": "MIT",
   "main": "dist/index.js",
@@ -26,11 +26,11 @@
   ],
   "dependencies": {
     "@duckdb/duckdb-wasm": "^1.28.0",
-    "@kepler.gl/common-utils": "3.1.0",
-    "@kepler.gl/constants": "3.1.0",
-    "@kepler.gl/processors": "3.1.0",
-    "@kepler.gl/table": "3.1.0",
-    "@kepler.gl/types": "3.1.0",
+    "@kepler.gl/common-utils": "3.1.1",
+    "@kepler.gl/constants": "3.1.1",
+    "@kepler.gl/processors": "3.1.1",
+    "@kepler.gl/table": "3.1.1",
+    "@kepler.gl/types": "3.1.1",
     "@monaco-editor/react": "^4.6.0",
     "@radix-ui/react-collapsible": "^1.1.0",
     "apache-arrow": ">=15.0.0",

package/src/duckdb/src/components/monaco-editor.tsx

@@ -92,7 +92,7 @@ const MonacoEditor: React.FC<MonacoEditorProps> = ({
       });
 
       monaco.languages.registerCompletionItemProvider('*', {
-        provideCompletionItems: (model, position, context, cancelationToken) => {
+        provideCompletionItems: (model, position, _context, _cancelationToken) => {
           const suggestions: monaco.languages.CompletionItem[] = [
             {
               label: 'myCustomSnippet',
@@ -152,7 +152,7 @@ const MonacoEditor: React.FC<MonacoEditorProps> = ({
                 ...tableSchema
                   .filter(d => d.table_name === table_name)
                   .map(
-                    ({table_name, column_name}) =>
+                    ({column_name}) =>
                       ({
                         label: column_name,
                         kind: monaco.languages.CompletionItemKind.Field,
@@ -170,7 +170,7 @@ const MonacoEditor: React.FC<MonacoEditorProps> = ({
         }
       });
     },
-    [tableSchema]
+    [tableSchema, schemaTableNames, schemaTableNamesSet]
   );
 
   return (

package/src/duckdb/src/components/preview-data-panel.tsx

@@ -1,13 +1,13 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
+import * as arrow from 'apache-arrow';
 import React, {useCallback, useMemo, useState, CSSProperties} from 'react';
+import {withTheme} from 'styled-components';
 
 import {DataTable, renderedSize} from '@kepler.gl/components';
-import {parseFieldValue, createDataContainer} from '@kepler.gl/utils';
 import {arrowSchemaToFields} from '@kepler.gl/processors';
-import {DataForm} from '@kepler.gl/utils';
-import {withTheme} from 'styled-components';
+import {parseFieldValue, createDataContainer, DataForm} from '@kepler.gl/utils';
 
 type BaseComponentProps = {
   className?: string;
@@ -39,27 +39,34 @@ export type DataTableStyle = {
   optionsButton?: number;
 };
 
+export type QueryResult = {
+  table: arrow.Table;
+  tableDuckDBTypes: Record<string, string>;
+};
+
 export type PreviewDataPanelProps = BaseComponentProps & {
-  result: any;
+  result: QueryResult;
   rowsToCalculatePreview?: number;
   theme?: any;
   setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;
   defaultPinnedColumns?: string[];
   dataTableStyle: DataTableStyle;
-  onAddResultToMap: (result: any) => void;
+  onAddResultToMap: (result: QueryResult) => void;
 };
 
 const PreviewDataPanelWOTheme: React.FC<PreviewDataPanelProps> = ({
   result,
   rowsToCalculatePreview = DEFAULT_ROWS_TO_CALCULATE_PREVIEW,
-  setColumnDisplayFormat,
   defaultPinnedColumns = [],
   theme
 }) => {
   const [pinnedColumns, setPinnedColumns] = useState<string[]>(defaultPinnedColumns);
-  const fields = useMemo(() => arrowSchemaToFields(result.schema), [result.schema]);
+  const fields = useMemo(
+    () => arrowSchemaToFields(result.table, result.tableDuckDBTypes),
+    [result]
+  );
   const dataContainer = useMemo(() => {
-    const cols = [...Array(result.numCols).keys()].map(i => result.getChildAt(i));
+    const cols = [...Array(result.table.numCols).keys()].map(i => result.table.getChildAt(i));
 
     const dataContainer = createDataContainer(cols, {
       fields,

package/src/duckdb/src/components/schema-panel.tsx

@@ -4,11 +4,15 @@
 import React, {useCallback, useEffect, useState} from 'react';
 import {useSelector} from 'react-redux';
 import styled from 'styled-components';
-import {arrowDataTypeToFieldType} from '@kepler.gl/utils';
-import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
+
+import {LoadingSpinner, Icons} from '@kepler.gl/components';
+import {arrowSchemaToFields} from '@kepler.gl/processors';
 import {VisState} from '@kepler.gl/schemas';
+
 import {Tree, DatasetNode, ColumnNode, TreeNodeData} from './tree';
 import {getDuckDB} from '../init';
+import {getDuckDBColumnTypes, getDuckDBColumnTypesMap} from '../table/duckdb-table-utils';
 
 // TODO note that demo state is available in demo-app, but not when add modules to dependencies in a custom map
 type State = {
@@ -26,64 +30,127 @@ const StyledSchemaPanel = styled.div`
   font-size: 12px;
   padding: 12px;
   font-family: ${props => props.theme.fontFamily};
+  height: 100%;
+`;
+
+const StyledLoadingSpinnerWrapper = styled.div`
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100%;
 `;
 
-async function getColumnSchema(c, name) {
-  const columnResult = await c.query(`Select * from '${name}' LIMIT 1;`);
+async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: string) {
+  const columnResult = await connection.query(`Select * from '${tableName}' LIMIT 1;`);
+
+  const columnDescribe = await getDuckDBColumnTypes(connection, tableName);
+  const keplerFields = arrowSchemaToFields(columnResult, getDuckDBColumnTypesMap(columnDescribe));
 
   return {
-    key: name,
+    key: tableName,
     object: {
       type: 'dataset',
-      tableName: name
+      tableName: tableName
     },
-    children: columnResult.schema.fields.map(field => {
-      const isGeoArrowColumn = field.metadata.get('ARROW:extension:name')?.startsWith('geoarrow');
+    children: columnResult.schema.fields.map((field, fieldIndex) => {
       return {
         key: field.name,
         object: {
           type: 'column',
           name: field.name,
           arrowType: field.type,
-          fieldType: isGeoArrowColumn
-            ? ALL_FIELD_TYPES.geoarrow
-            : arrowDataTypeToFieldType(field.type)
+          fieldType: keplerFields[fieldIndex].type
         }
       };
     })
   };
 }
 
-function getSchemaSuggestion(result) {
+export type SchemaSuggestion = {column_name: string; table_name: string};
+
+function getSchemaSuggestion(result: {key: string; children: {key: string}[]}[]) {
   return result.reduce((accu, data) => {
     const columns = data.children.map(child => ({
       column_name: child.key,
       table_name: data.key
     }));
     return accu.concat(columns);
-  }, []);
+  }, [] as SchemaSuggestion[]);
 }
-export const SchemaPanel = ({setTableSchema}) => {
+
+type SchemaPanelProps = {
+  setTableSchema: (tableSchema: SchemaSuggestion[]) => void;
+  droppedFile: File | null;
+  schemaUpdateTrigger: number;
+};
+
+const StyledSchemaPanelDropMessage = styled.div`
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100%;
+  flex-direction: column;
+  text-align: center;
+
+  div {
+    margin: 5px;
+  }
+  .header {
+    font-size: 15px;
+  }
+  .bold {
+    font-weight: 700;
+  }
+`;
+
+const StyledAddIcon = styled(Icons.Add)`
+  display: inline;
+  margin-top: -3px;
+`;
+
+export const SchemaPanelDropMessage = () => {
+  return (
+    <StyledSchemaPanelDropMessage>
+      <div className="header">
+        <StyledAddIcon /> Add files to DuckDB
+      </div>
+      <div className="bold">Supported formats: </div>
+      <div>.csv, .json, .geojson, .parquet, .arrow</div>
+      <div>Files you add will stay local to your browser.</div>
+    </StyledSchemaPanelDropMessage>
+  );
+};
+
+export const SchemaPanel = ({
+  setTableSchema,
+  droppedFile,
+  schemaUpdateTrigger
+}: SchemaPanelProps) => {
   const [columnSchemas, setColumnSchemas] = useState<TreeNodeData<{type: string}>[]>([]);
   const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);
-  useEffect(() => {
-    getTableSchema();
-  }, [datasets]);
+
   const getTableSchema = useCallback(async () => {
     const db = await getDuckDB();
     const c = await db.connect();
 
     const tableResult = await c.query('SHOW TABLES;');
 
-    const tableNames = tableResult.getChildAt(0)?.toJSON();
+    const tableNames: string[] | undefined = tableResult.getChildAt(0)?.toJSON();
 
     const result = await Promise.all((tableNames || [])?.map(name => getColumnSchema(c, name)));
     const tableSchema = getSchemaSuggestion(result);
 
     setColumnSchemas(result);
     setTableSchema(tableSchema);
-    c.close();
-  }, [setColumnSchemas, setTableSchema]);
+    await c.close();
+
+    // schemaUpdateTrigger indicates possible change in DuckDB
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [setColumnSchemas, setTableSchema, schemaUpdateTrigger]);
+
+  useEffect(() => {
+    getTableSchema();
+  }, [datasets, droppedFile, getTableSchema]);
 
   return (
     <StyledSchemaPanel>
@@ -102,8 +169,12 @@ export const SchemaPanel = ({setTableSchema}) => {
             }}
           />
         ))
+      ) : droppedFile ? (
+        <StyledLoadingSpinnerWrapper>
+          <LoadingSpinner />
+        </StyledLoadingSpinnerWrapper>
       ) : (
-        <div>No tables found</div>
+        <SchemaPanelDropMessage />
       )}
     </StyledSchemaPanel>
   );

package/src/duckdb/src/components/sql-panel.tsx

@@ -2,21 +2,34 @@
 // Copyright contributors to the kepler.gl project
 
 import * as arrow from 'apache-arrow';
-import React, {useCallback, useState, useEffect} from 'react';
+import React, {useCallback, useState, useEffect, useRef} from 'react';
 import {useDispatch} from 'react-redux';
 import styled from 'styled-components';
 import {Panel, PanelGroup, PanelResizeHandle} from 'react-resizable-panels';
-import MonacoEditor from './monaco-editor';
-import {SchemaPanel} from './schema-panel';
-import {PreviewDataPanel} from './preview-data-panel';
-import {getDuckDB} from '../init';
-import {Button, IconButton, Tooltip} from '@kepler.gl/components';
-import {generateHashId} from '@kepler.gl/common-utils';
+
 import {addDataToMap} from '@kepler.gl/actions';
-import {Icons, LoadingSpinner} from '@kepler.gl/components';
+import {generateHashId} from '@kepler.gl/common-utils';
+import {Button, FileDrop, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';
+import {arrowSchemaToFields} from '@kepler.gl/processors';
 import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';
 import {isAppleDevice} from '@kepler.gl/utils';
-import {arrowSchemaToFields} from '@kepler.gl/processors';
+
+import MonacoEditor from './monaco-editor';
+import {SchemaPanel, SchemaSuggestion} from './schema-panel';
+import {PreviewDataPanel, QueryResult} from './preview-data-panel';
+import {getDuckDB} from '../init';
+import {
+  constructST_asWKBQuery,
+  getDuckDBColumnTypes,
+  getDuckDBColumnTypesMap,
+  getGeometryColumns,
+  setGeoArrowWKBExtension,
+  splitSqlStatements,
+  checkIsSelectQuery,
+  removeSQLComments,
+  tableFromFile,
+  SUPPORTED_DUCKDB_DROP_EXTENSIONS
+} from '../table/duckdb-table-utils';
 
 const StyledSqlPanel = styled.div`
   display: flex;
@@ -114,6 +127,20 @@ const StyledErrorContainer = styled.pre`
   overflow: auto;
 `;
 
+interface StyledDragPanelProps {
+  dragOver?: boolean;
+}
+
+const StyledFileDropArea = styled(FileDrop)<StyledDragPanelProps>`
+  height: 100%;
+  border-width: 1px;
+  border: 1px ${props => (props.dragOver ? 'solid' : 'dashed')}
+    ${props => (props.dragOver ? props.theme.subtextColorLT : 'transparent')};
+  .file-drop-target {
+    height: 100%;
+  }
+`;
+
 type SqlPanelProps = {
   initialSql?: string;
 };
@@ -125,14 +152,19 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
     const params = new URLSearchParams(window.location.search);
     return params.get('sql') || initialSql;
   });
-  const [result, setResult] = useState<null | arrow.Table>(null);
+  const [droppedFile, setDroppedFile] = useState<File | null>(null);
+  const [dragState, setDragState] = useState(false);
+  const [result, setResult] = useState<null | QueryResult>(null);
+  const [schemaUpdateTrigger, setSchemaUpdateTrigger] = useState<number>(0);
   const [error, setError] = useState<Error | null>(null);
   const [counter, setCounter] = useState(0);
-  const [tableSchema, setTableSchema] = useState([]);
+  const [tableSchema, setTableSchema] = useState<SchemaSuggestion[]>([]);
   const [isRunning, setIsRunning] = useState(false);
   const [isMac] = useState(() => isAppleDevice());
   const dispatch = useDispatch();
 
+  const droppedFileAreaRef = useRef(null);
+
   useEffect(() => {
     const currentUrl = new URL(window.location.href);
     if (sql) {
@@ -146,7 +178,8 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
   const runQuery = useCallback(async () => {
     setIsRunning(true);
     try {
-      if (!sql?.trim()) {
+      const adjustedQuery = sql ? removeSQLComments(sql) : null;
+      if (!adjustedQuery) {
         setError(new Error('Query is empty'));
         return;
       }
@@ -154,16 +187,55 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
       const db = await getDuckDB();
       const connection = await db.connect();
 
-      const arrowResult = await connection.query(sql);
-      setResult(arrowResult);
-      setError(null);
+      // TODO find a cheap way to get DuckDb types with a single query to a remote resource - temp table? cte?
+      const tempTableName = 'temp_keplergl_table';
+
+      // remove comments
+      const sqlStatements = splitSqlStatements(adjustedQuery);
+
+      let arrowResult: arrow.Table | null = null;
+      let tableDuckDBTypes = {};
+
+      for (const statement of sqlStatements) {
+        const isLastQuery = statement === sqlStatements[sqlStatements.length - 1];
+        if (isLastQuery && (await checkIsSelectQuery(connection, statement))) {
+          // 1) create temp table from the original query.
+          await connection.query(`CREATE OR REPLACE TABLE '${tempTableName}' AS ${statement}`);
+
+          // 2) query duckdb types and detect candidate columns for ST_asWKB transform.
+          const duckDbColumns = await getDuckDBColumnTypes(connection, tempTableName);
+          tableDuckDBTypes = getDuckDBColumnTypesMap(duckDbColumns);
+          const columnsToConvertToWKB = getGeometryColumns(duckDbColumns);
+
+          // 3) query GEOMETRY columns as WKB.
+          const adjustedQuery = constructST_asWKBQuery(tempTableName, columnsToConvertToWKB);
+          arrowResult = await connection.query(adjustedQuery);
+
+          // 4) set geoarrow extension for the arrow table as DuckDB doesn't support the geoarrow extension.
+          setGeoArrowWKBExtension(arrowResult, duckDbColumns);
 
-      connection.close();
+          // 5) remove temp table
+          await connection.query(`DROP TABLE ${tempTableName};`);
+        } else {
+          await connection.query(statement);
+        }
+      }
+
+      // Show preview only for the result of the last query
+      if (arrowResult) {
+        setResult({table: arrowResult, tableDuckDBTypes});
+        setError(null);
+      }
+
+      await connection.close();
     } catch (e) {
       setError(e as Error);
     } finally {
       setIsRunning(false);
     }
+
+    // Indicate that there may be a possible change in DuckDB.
+    setSchemaUpdateTrigger(Date.now());
   }, [sql]);
 
   const onChange = useCallback(
@@ -176,13 +248,13 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
   const onAddResultToMap = useCallback(() => {
     if (!result) return;
 
-    const keplerFields = arrowSchemaToFields(result.schema);
+    const keplerFields = arrowSchemaToFields(result.table, result.tableDuckDBTypes);
 
     const datasetToAdd = {
       data: {
         fields: keplerFields,
         // TODO type AddDataToMapPayload -> rows -> + arrow.Table
-        rows: result as any
+        rows: result.table as any
       },
       info: {
         id: generateHashId(),
@@ -195,11 +267,77 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
     setCounter(counter + 1);
   }, [result, counter, dispatch]);
 
+  const isValidFileType = useCallback(filename => {
+    const fileExt = SUPPORTED_DUCKDB_DROP_EXTENSIONS.find(ext => filename.endsWith(ext));
+    return Boolean(fileExt);
+  }, []);
+
+  const createTableFromDroppedFile = useCallback(async (droppedFile: File | null) => {
+    if (droppedFile) {
+      const error = await tableFromFile(droppedFile);
+      if (error) {
+        setError(error);
+      } else {
+        setError(null);
+      }
+    }
+
+    setDroppedFile(null);
+    setDragState(false);
+  }, []);
+
+  useEffect(() => {
+    createTableFromDroppedFile(droppedFile);
+  }, [droppedFile, createTableFromDroppedFile]);
+
+  const handleFileInput = useCallback(
+    (fileList: FileList, event: DragEvent) => {
+      if (event) {
+        event.preventDefault();
+        event.stopPropagation();
+      }
+
+      const files = [...fileList].filter(Boolean);
+
+      const disableExtensionFilter = false;
+
+      const filesToLoad: File[] = [];
+      const errorFiles: string[] = [];
+      for (const file of files) {
+        if (disableExtensionFilter || isValidFileType(file.name)) {
+          filesToLoad.push(file);
+        } else {
+          errorFiles.push(file.name);
+        }
+      }
+
+      if (filesToLoad.length > 0) {
+        setDroppedFile(filesToLoad[0]);
+      } else if (errorFiles.length > 0) {
+        setError(new Error(`Unsupported file formats: ${errorFiles.join(', ')}`));
+      }
+    },
+    [isValidFileType]
+  );
+
   return (
     <StyledSqlPanel>
       <PanelGroup direction="horizontal">
-        <Panel defaultSize={20} minSize={15} style={SCHEMA_PANEL_STYLE}>
-          <SchemaPanel setTableSchema={setTableSchema} />
+        <Panel defaultSize={20} minSize={15} style={SCHEMA_PANEL_STYLE} className="schema-panel">
+          <StyledFileDropArea
+            dragOver={dragState}
+            onDragOver={() => setDragState(true)}
+            onDragLeave={() => setDragState(false)}
+            frame={droppedFileAreaRef.current || document}
+            onDrop={handleFileInput}
+            className="file-uploader__file-drop"
+          >
+            <SchemaPanel
+              setTableSchema={setTableSchema}
+              droppedFile={droppedFile}
+              schemaUpdateTrigger={schemaUpdateTrigger}
+            />
+          </StyledFileDropArea>
         </Panel>
 
         <StyledResizeHandle />
@@ -235,7 +373,7 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
               </Panel>
 
               <StyledVerticalResizeHandle />
-              <Panel>
+              <Panel className="preview-panel">
                 {isRunning ? (
                   <StyledLoadingContainer>
                     <LoadingSpinner />
@@ -250,7 +388,7 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
                       <Button secondary onClick={onAddResultToMap}>
                         Add to Map
                       </Button>
-                      <div>{result.numRows} rows</div>
+                      <div>{result.table.numRows} rows</div>
                     </StyledResultActions>
                     <PreviewDataPanel
                       result={result}

package/src/duckdb/src/init.ts

@@ -38,9 +38,12 @@ const _initializeDuckDb = async (config?: DuckDBConfig): Promise<AsyncDuckDB> =>
   // Select a bundle based on browser checks
   const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
   const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+  if (!bundle.mainWorker) {
+    throw new Error('Failed to initialize DuckDB');
+  }
 
   const worker_url = URL.createObjectURL(
-    new Blob([`importScripts("${bundle.mainWorker!}");`], {
+    new Blob([`importScripts("${bundle.mainWorker}");`], {
       type: 'text/javascript'
     })
   );

package/src/duckdb/src/processors/data-processor.ts

@@ -38,8 +38,8 @@ enum COLUMN_TYPES {
   TIMESTAMP = 'TIMESTAMP',
   VARCHAR = 'VARCHAR'
 }
-// @ts-expect-error https://duckdb.org/docs/data/csv/auto_detection.html#type-detection
-const COLUMN_TYPES_PRIORITIES = [
+// https://duckdb.org/docs/data/csv/auto_detection.html#type-detection
+export const COLUMN_TYPES_PRIORITIES = [
   'GEOMETRY',
   'BOOLEAN',
   'INTEGER',