@visactor/vquery 0.3.13 → 0.3.14

package/dist/node/esm/node.js

@@ -84,12 +84,12 @@ const applyWhere = (where)=>{
             case 'between':
                 {
                     const [a, b] = value;
-                    return external_kysely_sql`${external_kysely_sql.ref(field)} between (${external_kysely_sql.val(a)}, ${external_kysely_sql.val(b)})`;
+                    return external_kysely_sql`${external_kysely_sql.ref(field)} between ${external_kysely_sql.val(a)} and ${external_kysely_sql.val(b)}`;
                 }
             case 'not between':
                 {
                     const [a, b] = value;
-                    return external_kysely_sql`not ${external_kysely_sql.ref(field)} between (${external_kysely_sql.val(a)}, ${external_kysely_sql.val(b)})`;
+                    return external_kysely_sql`${external_kysely_sql.ref(field)} not between ${external_kysely_sql.val(a)} and ${external_kysely_sql.val(b)}`;
                 }
             default:
                 return external_kysely_sql`${external_kysely_sql.ref(field)} ${external_kysely_sql.raw(leaf.op)} ${external_kysely_sql.val(value)}`;
@@ -108,33 +108,62 @@ const applyLimit = (qb, limit)=>{
     if (limit && 'number' == typeof limit) qb = qb.limit(limit);
     return qb;
 };
-const convertDSLToSQL = (dsl, tableName)=>{
-    const db = new Kysely({
-        dialect: new PostgresDialect()
-    });
-    let qb = db.selectFrom(tableName);
-    qb = dsl.select && dsl.select.length > 0 ? qb.select((eb)=>dsl.select.map((item)=>{
+const DATE_FORMAT_MAP = {
+    year: '%Y',
+    month: '%Y-%m',
+    day: '%Y-%m-%d',
+    week: '%Y-W%W',
+    hour: '%Y-%m-%d %H',
+    minute: '%Y-%m-%d %H:%M',
+    second: '%Y-%m-%d %H:%M:%S'
+};
+const applySelect = (qb, select)=>{
+    if (select && select.length > 0) return qb.select((eb)=>select.map((item)=>{
             if (isSelectItem(item)) {
                 const field = item.field;
-                if (item.func) {
+                const expression = eb.ref(field);
+                if (item.aggr) {
+                    const { func } = item.aggr;
                     const alias = item.alias ?? field;
-                    switch(item.func){
-                        case 'avg':
-                            return eb.fn.avg(field).as(alias);
-                        case 'sum':
-                            return eb.fn.sum(field).as(alias);
-                        case 'min':
-                            return eb.fn.min(field).as(alias);
-                        case 'max':
-                            return eb.fn.max(field).as(alias);
-                        case 'count':
-                            return eb.fn.count(field).as(alias);
+                    if ([
+                        'avg',
+                        'sum',
+                        'min',
+                        'max',
+                        'variance',
+                        'variancePop',
+                        'stddev',
+                        'median'
+                    ].includes(func)) {
+                        if ('variance' === func) return external_kysely_sql`var_samp(${expression})`.as(alias);
+                        if ('variancePop' === func) return external_kysely_sql`var_pop(${expression})`.as(alias);
+                        return external_kysely_sql`${external_kysely_sql.raw(func)}(${expression})`.as(alias);
+                    }
+                    if ('count' === func) return external_kysely_sql`CAST(count(${expression}) AS INTEGER)`.as(alias);
+                    if ('quantile' === func) {
+                        const q = item.aggr.quantile ?? 0.5;
+                        return external_kysely_sql`quantile(${expression}, ${q})`.as(alias);
+                    } else if ('count_distinct' === func) return external_kysely_sql`CAST(count(distinct ${expression}) AS INTEGER)`.as(alias);
+                    else if (func.startsWith('to_')) {
+                        const dateTrunc = func.replace('to_', '');
+                        const format = DATE_FORMAT_MAP[dateTrunc];
+                        if (format) return external_kysely_sql`strftime(CAST(${expression} AS TIMESTAMP), ${format})`.as(alias);
+                        if ('quarter' === dateTrunc) return external_kysely_sql`strftime(CAST(${expression} AS TIMESTAMP), '%Y') || '-Q' || date_part('quarter', CAST(${expression} AS TIMESTAMP))`.as(alias);
                     }
                 }
-                return item.alias ? eb.ref(field).as(item.alias) : field;
+                const alias = item.alias ?? field;
+                return expression.as(alias);
             }
             return item;
-        })) : qb.selectAll();
+        }));
+    return qb.selectAll();
+};
+const convertDSLToSQL = (dsl, tableName)=>{
+    const db = new Kysely({
+        dialect: new PostgresDialect()
+    });
+    let qb = db.selectFrom(tableName);
+    qb = applySelect(qb, dsl.select);
     if (dsl.where) qb = qb.where(applyWhere(dsl.where));
     qb = applyGroupBy(qb, dsl.groupBy);
     if (dsl.orderBy && dsl.orderBy.length > 0) for (const o of dsl.orderBy)qb = qb.orderBy(o.field, o.order ?? 'asc');
@@ -142,60 +171,60 @@ const convertDSLToSQL = (dsl, tableName)=>{
     const compiled = qb.compile();
     return inlineParameters(compiled.sql, compiled.parameters);
 };
+const READ_FUNCTION_MAP = {
+    csv: 'read_csv_auto',
+    json: 'read_json_auto',
+    parquet: 'read_parquet'
+};
+const DATA_TYPE_MAP = {
+    number: 'DOUBLE',
+    string: 'VARCHAR',
+    date: 'DATE',
+    datetime: 'TIMESTAMP',
+    timestamp: 'TIMESTAMP'
+};
 class Dataset {
     queryAdapter;
     storageAdapter;
     _datasetId;
-    constructor(duckDB, indexedDB, datasetId){
-        this.queryAdapter = duckDB;
-        this.storageAdapter = indexedDB;
+    constructor(queryAdapter, storageAdapter, datasetId){
+        this.queryAdapter = queryAdapter;
+        this.storageAdapter = storageAdapter;
         this._datasetId = datasetId;
     }
     async init(temporaryColumns, temporaryDatasetSource) {
         const datasetInfo = await this.storageAdapter.readDataset(this._datasetId);
         if (!datasetInfo) throw new Error(`Dataset ${this._datasetId} not found`);
-        const columns = temporaryColumns ? temporaryColumns : datasetInfo.datasetSchema.columns;
-        const datasetSource = temporaryDatasetSource || datasetInfo.datasetSource;
+        const columns = temporaryColumns ?? datasetInfo.datasetSchema.columns;
+        const datasetSource = temporaryDatasetSource ?? datasetInfo.datasetSource;
         if (columns.length > 0 && datasetSource) await this.createOrReplaceView(columns, datasetSource);
     }
     async createOrReplaceView(columns, datasetSource) {
-        const readFunctionMap = {
-            csv: 'read_csv_auto',
-            json: 'read_json_auto',
-            xlsx: 'read_excel',
-            parquet: 'read_parquet'
-        };
-        const dataTypeMap = {
-            number: 'DOUBLE',
-            string: 'VARCHAR',
-            date: 'DATE',
-            datetime: 'TIMESTAMP',
-            timestamp: 'TIMESTAMP'
-        };
-        if (datasetSource) {
-            const readFunction = readFunctionMap[datasetSource.type];
-            if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
-            await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
-            const columnsStruct = `{${columns.map((c)=>`'${c.name}': '${dataTypeMap[c.type] || 'VARCHAR'}'`).join(', ')}}`;
-            const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
-            const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
-            await this.queryAdapter.query(createViewSql);
-        }
+        if (!datasetSource) return;
+        const readFunction = READ_FUNCTION_MAP[datasetSource.type];
+        if (!readFunction) throw new Error(`Unsupported dataSource type: ${datasetSource.type}`);
+        await this.queryAdapter.writeFile(this._datasetId, datasetSource.blob);
+        const columnsStruct = this.buildColumnsStruct(columns);
+        const columnNames = columns.map((c)=>`"${c.name}"`).join(', ');
+        let readSql = `${readFunction}('${this._datasetId}')`;
+        if ('csv' === datasetSource.type || 'json' === datasetSource.type) readSql = `${readFunction}('${this._datasetId}', columns=${columnsStruct})`;
+        const createViewSql = `CREATE OR REPLACE VIEW "${this._datasetId}" AS SELECT ${columnNames} FROM ${readSql}`;
+        await this.queryAdapter.query(createViewSql);
     }
     async query(queryDSL) {
         const sql = convertDSLToSQL(queryDSL, this.datasetId);
         return this.queryBySQL(sql);
     }
     async queryBySQL(sql) {
-        const start = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
+        const start = performance?.now?.() ?? Date.now();
         const result = await this.queryAdapter.query(sql);
-        const end = performance?.now?.()?.toFixed(3) ?? Date.now().toFixed(3);
+        const end = performance?.now?.() ?? Date.now();
         return {
             ...result,
             performance: {
-                startAt: start,
-                endAt: end,
-                duration: Number(end) - Number(start)
+                startAt: start.toFixed(3),
+                endAt: end.toFixed(3),
+                duration: end - start
             }
         };
     }
@@ -205,10 +234,22 @@ class Dataset {
     get datasetId() {
         return this._datasetId;
     }
+    buildColumnsStruct(columns) {
+        const columnDefs = columns.map((c)=>{
+            const duckDBType = DATA_TYPE_MAP[c.type] || 'VARCHAR';
+            return `'${c.name}': '${duckDBType}'`;
+        });
+        return `{${columnDefs.join(', ')}}`;
+    }
 }
 const isUrl = (url)=>isHttpUrl(url) || isBase64Url(url);
 const isHttpUrl = (url)=>url.startsWith('http://') || url.startsWith('https://');
 const isBase64Url = (url)=>url.startsWith('data:');
+const MIME_TYPES = {
+    csv: 'text/csv',
+    json: 'application/json',
+    parquet: 'application/parquet'
+};
 class DatasetSourceBuilder {
     type;
     value;
@@ -220,84 +261,30 @@ class DatasetSourceBuilder {
         return new DatasetSourceBuilder(raw);
     }
     async build() {
-        const blob = await DatasetSourceBuilder.convertToBlob(this.type, this.value);
+        const blob = await this.convertToBlob(this.type, this.value);
         return {
             type: this.type,
             blob: blob
         };
     }
-    static async convertToBlob(type, value) {
+    async convertToBlob(type, value) {
         if (value instanceof Blob) return value;
-        const convertCsvToBlob = (csvSource)=>{
-            if (csvSource instanceof ArrayBuffer) return new Blob([
-                csvSource
-            ], {
-                type: 'text/csv'
-            });
-            if ('string' == typeof csvSource && isUrl(csvSource)) return DatasetSourceBuilder.fetchBlob(csvSource);
-            return new Blob([
-                JSON.stringify(csvSource)
-            ], {
-                type: 'text/csv'
-            });
-        };
-        const convertJsonToBlob = (jsonSource)=>{
-            if (jsonSource instanceof ArrayBuffer) return new Blob([
-                jsonSource
-            ], {
-                type: 'application/json'
-            });
-            if ('string' == typeof jsonSource && isUrl(jsonSource)) return DatasetSourceBuilder.fetchBlob(jsonSource);
-            return new Blob([
-                JSON.stringify(jsonSource)
-            ], {
-                type: 'application/json'
-            });
-        };
-        const convertParquetToBlob = (parquetSource)=>{
-            if (parquetSource instanceof ArrayBuffer) return new Blob([
-                parquetSource
-            ], {
-                type: 'application/parquet'
-            });
-            if ('string' == typeof parquetSource && isUrl(parquetSource)) return DatasetSourceBuilder.fetchBlob(parquetSource);
-            return new Blob([
-                parquetSource
-            ], {
-                type: 'application/parquet'
-            });
-        };
-        const convertXlsxToBlob = (xlsxSource)=>{
-            if (xlsxSource instanceof ArrayBuffer) return new Blob([
-                xlsxSource
-            ], {
-                type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
-            });
-            if ('string' == typeof xlsxSource && isUrl(xlsxSource)) return DatasetSourceBuilder.fetchBlob(xlsxSource);
-            return new Blob([
-                xlsxSource
-            ], {
-                type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
-            });
-        };
-        switch(type){
-            case 'csv':
-                return convertCsvToBlob(value);
-            case 'json':
-                return convertJsonToBlob(value);
-            case 'xlsx':
-                return convertXlsxToBlob(value);
-            case 'parquet':
-                return convertParquetToBlob(value);
-            default:
-                return new Blob([
-                    value
-                ]);
+        if ('string' == typeof value && isUrl(value)) {
+            const response = await fetch(value);
+            return await response.blob();
         }
-    }
-    static async fetchBlob(url) {
-        const response = await fetch(url);
-        return await response.blob();
+        const mimeType = MIME_TYPES[type] || 'text/plain';
+        if (value instanceof ArrayBuffer) return new Blob([
+            value
+        ], {
+            type: mimeType
+        });
+        const content = 'object' == typeof value ? JSON.stringify(value) : String(value);
+        return new Blob([
+            content
+        ], {
+            type: mimeType
+        });
     }
 }
 class InMemoryAdapter {
@@ -306,47 +293,31 @@ class InMemoryAdapter {
     constructor(){}
     open = async ()=>{
         this.isOpen = true;
-        if (!this.isOpen) this.datasets = new Map();
+        this.datasets = new Map();
     };
     close = async ()=>{
         this.isOpen = false;
         this.datasets.clear();
     };
-    writeDataset = (datasetId, datasetSchema, datasetSource)=>new Promise((resolve, reject)=>{
-            try {
-                const record = {
-                    datasetId,
-                    datasetSchema,
-                    datasetSource
-                };
-                this.datasets.set(datasetId, record);
-                resolve();
-            } catch (error) {
-                reject(error);
-            }
+    writeDataset = (datasetId, datasetSchema, datasetSource)=>new Promise((resolve)=>{
+            const record = {
+                datasetId,
+                datasetSchema,
+                datasetSource
+            };
+            this.datasets.set(datasetId, record);
+            resolve();
         });
-    readDataset = (datasetId)=>new Promise((resolve, reject)=>{
-            try {
-                const record = this.datasets.get(datasetId);
-                resolve(record ? record : null);
-            } catch (error) {
-                reject(error);
-            }
+    readDataset = (datasetId)=>new Promise((resolve)=>{
+            const record = this.datasets.get(datasetId);
+            resolve(record ? record : null);
         });
-    deleteDataset = (datasetId)=>new Promise((resolve, reject)=>{
-            try {
-                this.datasets.delete(datasetId);
-                resolve();
-            } catch (error) {
-                reject(error);
-            }
+    deleteDataset = (datasetId)=>new Promise((resolve)=>{
+            this.datasets.delete(datasetId);
+            resolve();
         });
-    listDatasets = ()=>new Promise((resolve, reject)=>{
-            try {
-                resolve(Array.from(this.datasets.values()));
-            } catch (error) {
-                reject(error);
-            }
+    listDatasets = ()=>new Promise((resolve)=>{
+            resolve(Array.from(this.datasets.values()));
         });
 }
 class DuckDBNodeQueryAdapter {
@@ -409,9 +380,9 @@ class VQuery {
     queryAdapter;
     storageAdapter;
     isInitialized = false;
-    constructor(){
-        this.queryAdapter = new DuckDBNodeQueryAdapter();
-        this.storageAdapter = new InMemoryAdapter();
+    constructor(queryAdapter, storageAdapter){
+        this.queryAdapter = queryAdapter;
+        this.storageAdapter = storageAdapter;
     }
     async checkInitialized() {
         if (!this.isInitialized) {
@@ -473,4 +444,9 @@ class VQuery {
         await this.storageAdapter.close();
     }
 }
-export { DatasetSourceBuilder, VQuery, convertDSLToSQL, isBase64Url, isHttpUrl, isUrl };
+class vquery_node_VQuery extends VQuery {
+    constructor(){
+        super(new DuckDBNodeQueryAdapter(), new InMemoryAdapter());
+    }
+}
+export { Dataset, DatasetSourceBuilder, DuckDBNodeQueryAdapter, InMemoryAdapter, vquery_node_VQuery as VQuery, convertDSLToSQL, isBase64Url, isHttpUrl, isUrl };

package/dist/node/esm/sql-builder/builders/index.d.ts

@@ -1,3 +1,4 @@
 export { applyWhere } from './where';
 export { applyGroupBy } from './groupBy';
 export { applyLimit } from './limit';
+export { applySelect } from './select';

package/dist/node/esm/sql-builder/builders/select.d.ts

@@ -0,0 +1,3 @@
+import { SelectItem } from '../../types/dsl/Select';
+import type { SelectQueryBuilder } from 'kysely';
+export declare const applySelect: <DB, TB extends keyof DB & string, O, T>(qb: SelectQueryBuilder<DB, TB, O>, select?: Array<keyof T | SelectItem<T>>) => SelectQueryBuilder<DB, TB, O & import("kysely").Selection<DB, TB, any>>;

package/dist/node/esm/sql-builder/dslToSQL.d.ts

@@ -1,2 +1,2 @@
-import { QueryDSL } from '../types';
-export declare const convertDSLToSQL: <T, TableName extends string>(dsl: QueryDSL<T>, tableName: TableName) => string;
+import { QueryDSL, VQueryDSL } from '../types';
+export declare const convertDSLToSQL: <T, TableName extends string>(dsl: QueryDSL<T> | VQueryDSL<T>, tableName: TableName) => string;

package/dist/node/esm/types/DataSource.d.ts

@@ -1,5 +1,5 @@
 export type TidyDatum = Record<string, number | string | null | boolean | undefined>;
-export type DatasetSourceType = 'csv' | 'json' | 'xlsx' | 'parquet';
+export type DatasetSourceType = 'csv' | 'json' | 'parquet';
 export type DatasetSourceValue = string | ArrayBuffer | Blob | TidyDatum[];
 export interface DatasetSource {
     type: DatasetSourceType;

package/dist/node/esm/types/dsl/GroupBy.d.ts

@@ -1 +1 @@
-export type GroupBy<T> = Array<keyof T>;
+export type GroupBy<T> = Array<keyof T | (string & {})>;

package/dist/node/esm/types/dsl/Select.d.ts

@@ -1,7 +1,12 @@
-export type AggregateFunction = 'count' | 'sum' | 'avg' | 'min' | 'max' | 'quantile';
+export type BaseAggregateFunction = 'count' | 'count_distinct' | 'sum' | 'avg' | 'min' | 'max' | 'variance' | 'variancePop' | 'stddev' | 'median' | 'quantile';
+export type DateAggregateFunction = 'to_year' | 'to_quarter' | 'to_month' | 'to_week' | 'to_day' | 'to_hour' | 'to_minute' | 'to_second';
+export type AggregateFunction = BaseAggregateFunction | DateAggregateFunction;
 export type SelectItem<T> = {
     field: keyof T;
     alias?: string;
-    func?: AggregateFunction;
+    aggr?: {
+        func: AggregateFunction;
+        quantile?: number;
+    };
 };
 export type Select<T> = Array<keyof T | SelectItem<T>>;

package/dist/node/esm/vquery-browser.d.ts

@@ -1,21 +1,4 @@
-import { Dataset } from './dataset/dataset';
-import { RawDatasetSource, DatasetColumn } from './types';
-export declare class VQuery {
-    private queryAdapter;
-    private storageAdapter;
-    private isInitialized;
+import { VQuery as VQueryCore } from './VQuery';
+export declare class VQuery extends VQueryCore {
     constructor();
-    private checkInitialized;
-    private checkDatasetExists;
-    createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
-    updateDatasetSource(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
-    dropDataset(datasetId: string): Promise<void>;
-    connectDataset(datasetId: string, temporaryColumns?: DatasetColumn[], temporaryRawDatasetSource?: RawDatasetSource): Promise<Dataset>;
-    hasDataset(datasetId: string): Promise<boolean>;
-    listDatasets(): Promise<{
-        datasetId: string;
-        datasetSchema: import("./types").DatasetSchema;
-        datasetSource?: import("./types").DatasetSource;
-    }[]>;
-    close(): Promise<void>;
 }

package/dist/node/esm/vquery-node.d.ts

@@ -1,21 +1,4 @@
-import { Dataset } from './dataset/dataset';
-import { RawDatasetSource, DatasetColumn } from './types';
-export declare class VQuery {
-    private queryAdapter;
-    private storageAdapter;
-    private isInitialized;
+import { VQuery as VQueryCore } from './VQuery';
+export declare class VQuery extends VQueryCore {
     constructor();
-    private checkInitialized;
-    private checkDatasetExists;
-    createDataset(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
-    updateDatasetSource(datasetId: string, columns?: DatasetColumn[], rawDatasetSource?: RawDatasetSource): Promise<void>;
-    dropDataset(datasetId: string): Promise<void>;
-    connectDataset(datasetId: string, temporaryColumns?: DatasetColumn[], temporaryRawDatasetSource?: RawDatasetSource): Promise<Dataset>;
-    hasDataset(datasetId: string): Promise<boolean>;
-    listDatasets(): Promise<{
-        datasetId: string;
-        datasetSchema: import("./types").DatasetSchema;
-        datasetSource?: import("./types").DatasetSource;
-    }[]>;
-    close(): Promise<void>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@visactor/vquery",
-  "version": "0.3.13",
+  "version": "0.3.14",
   "type": "module",
   "exports": {
     ".": {
@@ -57,11 +57,14 @@
   },
   "scripts": {
     "build": "rslib build",
+    "build:test": "node scripts/build-tests.js",
+    "build:docs": "node scripts/build-docs.js",
+    "g": "pnpm run build:test && pnpm run build:docs",
     "dev": "rslib build --watch --no-clean",
     "lint": "eslint .",
     "test": "rstest",
     "test:update": "rstest --update",
-    "test:coverage": "rstest --coverage; open ./coverage/index.html",
+    "test:coverage": "rstest --coverage && node scripts/build-coverage-badge.mjs && open ./coverage/index.html",
     "typecheck": "tsc --noEmit"
   }
 }

package/dist/browser/esm/types/dsl/demo.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/node/cjs/types/dsl/demo.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/node/esm/types/dsl/demo.d.ts

@@ -1 +0,0 @@
-export {};