@perspective-dev/client 4.2.0 → 4.4.0

package/src/ts/virtual_servers/duckdb.ts

@@ -76,60 +76,47 @@ const FILTER_OPS = [
 ];
 
 function duckdbTypeToPsp(name: string): ColumnType {
-    if (name === "VARCHAR" || name == "Utf8") {
+    name = name.toLowerCase();
+    if (name === "varchar" || name == "utf8") {
         return "string";
     }
 
     if (
-        name === "DOUBLE" ||
-        name === "BIGINT" ||
-        name === "HUGEINT" ||
-        name === "Float64" ||
-        name.startsWith("Decimal")
+        name === "double" ||
+        name === "bigint" ||
+        name === "hugeint" ||
+        name === "float64" ||
+        name.startsWith("decimal")
     ) {
         return "float";
     }
 
-    if (name.startsWith("Int") || name == "INTEGER") {
+    if (name.startsWith("int")) {
         return "integer";
     }
 
-    if (name === "INTEGER") {
-        return "integer";
-    }
-
-    if (name === "DATE" || name.startsWith("Date")) {
+    if (name.startsWith("date")) {
         return "date";
     }
 
-    if (name === "BOOLEAN" || name === "Bool") {
+    if (name.startsWith("bool")) {
         return "boolean";
     }
 
-    if (name === "TIMESTAMP" || name.startsWith("Timestamp")) {
+    if (name.startsWith("timestamp")) {
         return "datetime";
     }
 
-    throw new Error(`Unknown type '${name}'`);
-}
-
-function convertDecimalToNumber(value: any, dtypeString: string) {
-    if (!(value instanceof Uint32Array || value instanceof Int32Array)) {
-        return value;
+    if (name.startsWith("json")) {
+        return "string";
     }
 
-    let bigIntValue = BigInt(0);
-    for (let i = 0; i < value.length; i++) {
-        bigIntValue |= BigInt(value[i]) << BigInt(i * 32);
+    if (name.startsWith("struct")) {
+        return "string";
     }
 
-    const scaleMatch = dtypeString.match(/Decimal\[\d+e(\d+)\]/);
-    if (scaleMatch) {
-        const scale = parseInt(scaleMatch[1]);
-        return Number(bigIntValue) / Math.pow(10, scale);
-    } else {
-        return Number(bigIntValue);
-    }
+    console.warn(`Unknown type '${name}'`);
+    return "string";
 }
 
 async function runQuery(
@@ -202,6 +189,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
             split_by: true,
             sort: true,
             expressions: true,
+            group_rollup_mode: ["rollup", "flat", "total"],
             filter_ops: {
                 integer: FILTER_OPS,
                 float: FILTER_OPS,
@@ -250,12 +238,10 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
     async viewColumnSize(viewId: string, config: ViewConfig) {
         const query = this.sqlBuilder.viewColumnSize(viewId);
         const results = await runQuery(this.db, query);
-        const gs = config.group_by?.length || 0;
         const count = Number(Object.values(results[0].toJSON())[0]);
-        return (
-            count -
-            (gs === 0 ? 0 : gs + (config.split_by?.length === 0 ? 1 : 0))
-        );
+        const gs = config.group_by?.length || 0;
+        const is_flat = config.group_rollup_mode === "flat";
+        return count - (gs === 0 ? 0 : is_flat ? gs : gs + 1);
     }
 
     async tableSize(tableId: string) {
@@ -285,6 +271,20 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
         await runQuery(this.db, query);
     }
 
+    async viewGetMinMax(
+        viewId: string,
+        columnName: string,
+        config: ViewConfig,
+    ) {
+        const query = this.sqlBuilder.viewGetMinMax(viewId, columnName, config);
+        const results = await runQuery(this.db, query);
+        const row = results[0].toJSON();
+        let [min, max] = Object.values(row);
+        if (typeof min === "bigint") min = Number(min);
+        if (typeof max === "bigint") max = Number(max);
+        return { min: min ?? null, max: max ?? null };
+    }
+
     async viewGetData(
         viewId: string,
         config: ViewConfig,
@@ -292,8 +292,6 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
         viewport: ViewWindow,
         dataSlice: perspective.VirtualDataSlice,
     ) {
-        const is_group_by = config.group_by?.length > 0;
-        const is_split_by = config.split_by?.length > 0;
         const query = this.sqlBuilder.viewGetData(
             viewId,
             config,
@@ -301,38 +299,10 @@ export class DuckDBHandler implements perspective.VirtualServerHandler {
             schema,
         );
 
-        const { rows, columns, dtypes } = await runQuery(this.db, query, {
-            columns: true,
-        });
-
-        for (let cidx = 0; cidx < columns.length; cidx++) {
-            if (cidx === 0 && is_group_by && !is_split_by) {
-                // This is the grouping_id column, skip it
-                continue;
-            }
-
-            let col = columns[cidx];
-            if (is_split_by && !col.startsWith("__ROW_PATH_")) {
-                col = col.replaceAll("_", "|");
-            }
-
-            const dtype = duckdbTypeToPsp(dtypes[cidx]) as ColumnType;
-
-            const isDecimal = dtypes[cidx].startsWith("Decimal");
-            for (let ridx = 0; ridx < rows.length; ridx++) {
-                const rowArray = rows[ridx].toArray();
-                const grouping_id = Number(rowArray[0]);
-                let value = rowArray[cidx];
-                if (isDecimal) {
-                    value = convertDecimalToNumber(value, dtypes[cidx]);
-                }
-
-                if (typeof value === "bigint") {
-                    value = Number(value);
-                }
+        const ipc = await this.db.useUnsafe((bindings, conn) =>
+            bindings.runQuery(conn, query),
+        );
 
-                dataSlice.setCol(dtype, col, ridx, value, grouping_id);
-            }
-        }
+        dataSlice.fromArrowIpc(ipc);
     }
 }

package/tsconfig.json

@@ -16,6 +16,7 @@
         "./src/ts/perspective.node.ts",
         "./src/ts/perspective.cdn.ts",
         "./src/ts/virtual_servers/duckdb.ts",
+        "./src/ts/virtual_servers/clickhouse.ts",
         "./test/js/*.ts"
     ]
 }