@kanaries/graphic-walker 0.3.16 → 0.4.0

package/src/fields/filterField/tabs.tsx

@@ -1,10 +1,13 @@
 import { observer } from 'mobx-react-lite';
-import React, { useMemo, useRef, useState } from 'react';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import styled from 'styled-components';
 
-import type { IFilterField, IFilterRule } from '../../interfaces';
+import type { IFilterField, IFilterRule, IRow, DataSet, IFieldStats, IField, IViewField } from '../../interfaces';
 import { useGlobalStore } from '../../store';
+import LoadingLayer from '../../components/loadingLayer';
+import { useComputationFunc, useRenderer } from '../../renderer/hooks';
+import { fieldStatServer } from '../../computation/serverComputation';
 import Slider from './slider';
 import {
     ChevronDownIcon,
@@ -12,6 +15,7 @@ import {
 } from '@heroicons/react/24/outline';
 
 export type RuleFormProps = {
+    dataset: DataSet;
     field: IFilterField;
     onChange: (rule: IFilterRule) => void;
 };
@@ -27,15 +31,15 @@ const Container = styled.div`
         > * {
             margin-inline-start: 0.6em;
 
-            &:first-child: {
+            &:first-child {
                 margin-inline-start: 0;
-            },
-        },
-    },
+            }
+        }
+    }
 `;
 
 export const Button = styled.button`
-    :hover: {
+    :hover {
         background-color: rgba(243, 244, 246, 0.5);
     };
     color: rgb(55, 65, 81);
@@ -127,13 +131,75 @@ const StatusCheckbox: React.FC<{ currentNum: number; totalNum: number; onChange:
     )
 }
 
+type FieldDistributionEntry = IFieldStats['values'][number];
+
+const defaultValueComparator = (a: any, b: any) => {
+    if (typeof a === 'number' && typeof b === 'number') {
+        return a - b;
+    } else {
+        return String(a).localeCompare(String(b))
+    }
+};
+
+const countCmp = (a: FieldDistributionEntry, b: FieldDistributionEntry) => {
+    return a.count - b.count;
+};
+
+const useFieldStats = (
+    field: IField,
+    attributes: { values: boolean; range: boolean },
+    sortBy: 'value' | 'value_dsc' | 'count' | 'count_dsc' | 'none'
+): IFieldStats | null => {    
+    const { values, range } = attributes;
+    const { fid, cmp = defaultValueComparator } = field;
+    const valueCmp = React.useCallback<typeof countCmp>((a, b) => {
+        return cmp(a.value, b.value);
+    }, [cmp]);
+    const comparator = sortBy === "none" ? null : sortBy.startsWith("value") ? valueCmp : countCmp;
+    const sortMulti = sortBy.endsWith("dsc") ? -1 : 1;
+    const [loading, setLoading] = React.useState(true);
+    const [stats, setStats] = React.useState<IFieldStats | null>(null);
+    const computationFunction = useComputationFunc();
+
+    React.useEffect(() => {
+        setLoading(true);
+        let isCancelled = false;
+        fieldStatServer(computationFunction, fid, { values, range }).then(stats => {
+            if (isCancelled) {
+                return;
+            }
+            setStats(stats);
+            setLoading(false);
+        }).catch(reason => {
+            console.warn(reason);
+            if (isCancelled) {
+                return;
+            }
+            setStats(null);
+            setLoading(false);
+        });
+        return () => {
+            isCancelled = true;
+        };
+    }, [fid, computationFunction, values, range]);
+
+    const sortedStats = React.useMemo<typeof stats>(() => {
+        if (!stats || !comparator) {
+            return stats;
+        }
+        const copy = { ...stats };
+        copy.values = copy.values.slice().sort((a,b) => sortMulti * comparator(a,b));
+        return copy;
+    }, [stats, comparator, sortMulti]);
+
+    return loading ? null : sortedStats;
+};
+
 export const FilterOneOfRule: React.FC<RuleFormProps & { active: boolean }> = observer(({
     active,
     field,
     onChange,
 }) => {
-    const { commonStore } = useGlobalStore();
-    const { currentDataset: { dataSource } } = commonStore;
 
     interface SortConfig {
         key: 'value' | 'count';
@@ -144,67 +210,41 @@ export const FilterOneOfRule: React.FC<RuleFormProps & { active: boolean }> = ob
         ascending: true
     });
 
-    const count = React.useMemo(() => {
-        return dataSource.reduce<Map<string | number, number>>((tmp, d) => {
-            const val = d[field.fid];
-
-            tmp.set(val, (tmp.get(val) ?? 0) + 1);
-
-            return tmp;
-        }, new Map<string | number, number>());
-    }, [dataSource, field]);
-
-    const sortedList = useMemo(() => {
-        const entries = Array.from(count.entries());
-        const compare = (a: [string | number, number], b: [string | number, number]) => {
-            if (sortConfig.key === 'count') {
-                return a[1] - b[1];
-            } else {
-                if (typeof a[0] === 'number' && typeof b[0] === 'number') {
-                    return a[0] - b[0];
-                } else {
-                    return String(a[0]).localeCompare(String(b[0]))
-                }
-            }
-        }
-        entries.sort(sortConfig.ascending ? compare : (a, b) => -compare(a, b));
-        return entries;
-    }, [count, sortConfig]);
-
     const { t } = useTranslation('translation');
 
+    const stats = useFieldStats(field, { values: true, range: false }, `${sortConfig.key}${sortConfig.ascending ? '': '_dsc'}`);
+    const count = stats?.values;
+
     React.useEffect(() => {
-        if (active && field.rule?.type !== 'one of') {
+        if (count && active && field.rule?.type !== 'one of') {
             onChange({
                 type: 'one of',
-                value: new Set<string | number>(count.keys()),
+                value: new Set<string | number>(count.map(item => item.value)),
             });
         }
     }, [active, onChange, field, count]);
 
     const handleToggleFullOrEmptySet = () => {
-        if (!field.rule || field.rule.type !== 'one of') return;
+        if (!field.rule || field.rule.type !== 'one of' || !count) return;
         const curSet = field.rule.value;
         onChange({
             type: 'one of',
             value: new Set<number | string>(
-                curSet.size === count.size
-                    ? []
-                    : count.keys()
+                curSet.size === count.length ? [] : count.map(c => c.value)
             ),
         });
     }
     const handleToggleReverseSet = () => {
-        if (!field.rule || field.rule.type !== 'one of') return;
+        if (!field.rule || field.rule.type !== 'one of' || !count) return;
         const curSet = field.rule.value;
         onChange({
             type: 'one of',
             value: new Set<number | string>(
-                [...count.keys()].filter(key => !curSet.has(key))
+                count.map(c => c.value).filter(key => !curSet.has(key))
             ),
         });
     }
-    const handleSelectValue = (value, checked) => {
+    const handleSelectValue = (value: any, checked: boolean) => {
         if (!field.rule || field.rule?.type !== 'one of') return;
         const rule: IFilterRule = {
             type: 'one of',
@@ -219,12 +259,20 @@ export const FilterOneOfRule: React.FC<RuleFormProps & { active: boolean }> = ob
     }
 
     const selectedValueSum = useMemo(() => {
-        if (!field.rule) return 0;
+        if (!field.rule?.value || !count) return 0;
         return [...field.rule.value].reduce<number>((sum, key) => {
-            const s = dataSource.filter(which => which[field.fid] === key).length;
+            const s = count.find(c => c.value === key)?.count || 0;
             return sum + s;
         }, 0)
-    }, [field.rule?.value]);
+    }, [field.rule?.value, count, field.fid]);
+
+    if (!stats) {
+        return (
+            <div className="h-24 w-full relative">
+                <LoadingLayer />
+            </div>
+        );
+    }
 
     const SortButton: React.FC<{ currentKey: SortConfig["key"] }> = ({ currentKey }) => {
         const isCurrentKey = sortConfig.key === currentKey;
@@ -249,9 +297,10 @@ export const FilterOneOfRule: React.FC<RuleFormProps & { active: boolean }> = ob
                 <Button
                     className="dark:bg-zinc-900 dark:text-gray-200 dark:hover:bg-gray-800"
                     onClick={() => handleToggleFullOrEmptySet()}
+                    disabled={!count}
                 >
                     {
-                        field.rule.value.size === count.size
+                        field.rule.value.size === count?.length
                             ? t('filters.btn.unselect_all')
                             : t('filters.btn.select_all')
                     }
@@ -267,7 +316,7 @@ export const FilterOneOfRule: React.FC<RuleFormProps & { active: boolean }> = ob
                 <div className="flex justify-center items-center">
                     <StatusCheckbox
                         currentNum={field.rule.value.size}
-                        totalNum={count.size}
+                        totalNum={count?.length ?? 0}
                         onChange={handleToggleFullOrEmptySet}
                     />
                 </div>
@@ -283,7 +332,7 @@ export const FilterOneOfRule: React.FC<RuleFormProps & { active: boolean }> = ob
             {/* <hr /> */}
             <Table>
                 {
-                    sortedList.map(([value, count], idx) => {
+                    count?.map(({ value, count }, idx) => {
                         const id = `rule_checkbox_${idx}`;
 
                         return (
@@ -339,9 +388,11 @@ interface CalendarInputProps {
 const CalendarInput: React.FC<CalendarInputProps> = props => {
     const { min, max, value, onChange } = props;
     const dateStringFormatter = (timestamp: number) => {
-        return new Date(timestamp).toISOString().slice(0, 19);
+        const date = new Date(timestamp);
+        if (Number.isNaN(date.getTime())) return '';
+        return date.toISOString().slice(0, 19);
     }
-    const handleSubmitDate = (value) => {
+    const handleSubmitDate = (value: string) => {
         if (new Date(value).getTime() <= max && new Date(value).getTime() >= min) {
             onChange(new Date(value).getTime())
         }
@@ -358,18 +409,48 @@ const CalendarInput: React.FC<CalendarInputProps> = props => {
     )
 }
 
+const emptyFilters = [] as const;
+
 export const FilterTemporalRangeRule: React.FC<RuleFormProps & { active: boolean }> = observer(({
+    dataset,
     active,
     field,
     onChange,
 }) => {
-    const { commonStore } = useGlobalStore();
-    const { currentDataset: { dataSource } } = commonStore;
+    const { rawFields } = dataset;
 
     const { t } = useTranslation('translation');
 
+    const fields = useMemo(() => {
+        return rawFields.map<Omit<IViewField, 'dragId'>>(f => ({
+            ...f,
+            name: f.name || f.fid,
+        }));
+    }, [rawFields]);
+
+    const viewDimensions = useMemo(() => {
+        return field.analyticType === 'dimension' ? [field] : [];
+    }, [field]);
+
+    const viewMeasures = useMemo(() => {
+        return field.analyticType === 'measure' ? [field] : [];
+    }, [field]);
+
+    const computationFunction = useComputationFunc();
+
+    const { viewData, loading } = useRenderer({
+        allFields: fields,
+        viewDimensions,
+        viewMeasures,
+        filters: emptyFilters,
+        defaultAggregated: false,
+        computationFunction,
+        limit: 1000,
+        sort: 'none',
+    });
+
     const sorted = React.useMemo(() => {
-        return dataSource.reduce<number[]>((list, d) => {
+        return viewData.reduce<number[]>((list, d) => {
             try {
                 const time = new Date(d[field.fid]).getTime();
 
@@ -379,20 +460,30 @@ export const FilterTemporalRangeRule: React.FC<RuleFormProps & { active: boolean
             }
             return list;
         }, []).sort((a, b) => a - b);
-    }, [dataSource, field]);
+    }, [viewData, field.fid]);
 
-    const [min, max] = React.useMemo(() => {
-        return [sorted[0] ?? 0, Math.max(sorted[sorted.length - 1] ?? 0, sorted[0] ?? 0)];
+    const [min, max, loaded] = React.useMemo<[min: number, max: number, loaded: boolean]>(() => {
+        if (!sorted.length) return [0, 0, false];
+        return [sorted[0] ?? 0, Math.max(sorted[sorted.length - 1] ?? 0, sorted[0] ?? 0), true];
     }, [sorted]);
 
     React.useEffect(() => {
         if (active && field.rule?.type !== 'temporal range') {
             onChange({
                 type: 'temporal range',
-                value: [sorted[0] ?? 0, Math.max(sorted[sorted.length - 1] ?? 0, sorted[0] ?? 0)],
+                value: [min, max],
             });
         }
-    }, [onChange, field, sorted, active]);
+    }, [onChange, field, min, max, active]);
+
+    React.useEffect(() => {
+        if (active && loaded && field.rule?.type === 'temporal range' && field.rule.value[0] !== min && field.rule.value[1] !== max) {
+            onChange({
+                type: 'temporal range',
+                value: [min, max],
+            });
+        }
+    }, [field.rule, min, max, active, loaded]);
 
     const handleChange = React.useCallback((value: readonly [number, number]) => {
         onChange({
@@ -401,6 +492,14 @@ export const FilterTemporalRangeRule: React.FC<RuleFormProps & { active: boolean
         });
     }, []);
 
+    if (loading) {
+        return (
+            <div className="h-24 w-full relative">
+                <LoadingLayer />
+            </div>
+        );
+    }
+
     return field.rule?.type === 'temporal range' ? (
         <Container className="overflow-visible">
             <div>{t('constant.filter_type.temporal_range')}</div>
@@ -434,27 +533,19 @@ export const FilterRangeRule: React.FC<RuleFormProps & { active: boolean }> = ob
     field,
     onChange,
 }) => {
-    const { commonStore } = useGlobalStore();
-    const { currentDataset: { dataSource } } = commonStore;
-
     const { t } = useTranslation('translation', { keyPrefix: 'constant.filter_type' });
 
-    const sorted = React.useMemo(() => {
-        return dataSource.map(d => d[field.fid]).sort((a, b) => a - b);
-    }, [dataSource, field]);
-
-    const [min, max] = React.useMemo(() => {
-        return [sorted[0] ?? 0, Math.max(sorted[sorted.length - 1] ?? 0, sorted[0] ?? 0)];
-    }, [sorted]);
+    const stats = useFieldStats(field, { values: false, range: true }, 'none');
+    const range = stats?.range;
 
     React.useEffect(() => {
-        if (active && field.rule?.type !== 'range') {
+        if (range && active && field.rule?.type !== 'range') {
             onChange({
                 type: 'range',
-                value: [min, max],
+                value: range,
             });
         }
-    }, [onChange, field, min, max, active]);
+    }, [onChange, field, range, active]);
 
     const handleChange = React.useCallback((value: readonly [number, number]) => {
         onChange({
@@ -463,13 +554,21 @@ export const FilterRangeRule: React.FC<RuleFormProps & { active: boolean }> = ob
         });
     }, []);
 
+    if (!range) {
+        return (
+            <div className="h-24 w-full relative">
+                <LoadingLayer />
+            </div>
+        );
+    }
+
     return field.rule?.type === 'range' ? (
         <Container>
             <div>{t('range')}</div>
             <div className="text-gray-500">{t('range_desc')}</div>
             <Slider
-                min={min}
-                max={max}
+                min={range[0]}
+                max={range[1]}
                 value={field.rule.value}
                 onChange={handleChange}
             />
@@ -503,8 +602,9 @@ export interface TabsProps extends RuleFormProps {
 }
 
 const Tabs: React.FC<TabsProps> = observer(({ field, onChange, tabs }) => {
-    const { vizStore } = useGlobalStore();
+    const { vizStore, commonStore } = useGlobalStore();
     const { draggableFieldState } = vizStore;
+    const { currentDataset } = commonStore;
 
     const { t } = useTranslation('translation', { keyPrefix: 'constant.filter_type' });
 
@@ -563,6 +663,7 @@ const Tabs: React.FC<TabsProps> = observer(({ field, onChange, tabs }) => {
                                     field={field}
                                     onChange={onChange}
                                     active={which === tab}
+                                    dataset={currentDataset}
                                 />
                             </TabItem>
                         );

package/src/index.tsx

@@ -38,3 +38,5 @@ export const GraphicWalker = observer(forwardRef<IGWHandler, IGWProps>((props, r
 export { default as PureRenderer } from './renderer/pureRenderer';
 export { embedGraphicWalker } from './vanilla';
 export type { IGWProps };
+export { ISegmentKey } from './interfaces';
+export { resolveSpecFromStoInfo } from './utils/save';

package/src/interfaces.ts

@@ -1,5 +1,6 @@
 import {Config as VgConfig, View} from 'vega';
 import {Config as VlConfig} from 'vega-lite';
+import type {IViewQuery} from "./lib/viewQuery";
 
 export type DeepReadonly<T extends Record<keyof any, any>> = {
     readonly [K in keyof T]: T[K] extends Record<keyof any, any> ? DeepReadonly<T[K]> : T[K];
@@ -52,6 +53,15 @@ export interface IUncertainMutField {
     path: string[];
 }
 
+export interface IDatasetStats {
+    rowCount: number;
+}
+
+export interface IFieldStats {
+    values: { value: number | string; count: number }[];
+    range: [number, number];
+}
+
 export type IExpParamter =
     | {
           type: 'field';
@@ -95,12 +105,12 @@ export interface IField {
     computed?: boolean;
     expression?: IExpression;
     basename?: string;
-    path?: [],
+    path?: string[],
 }
-
+export type ISortMode = 'none' | 'ascending' | 'descending';
 export interface IViewField extends IField {
     dragId: string;
-    sort?: 'none' | 'ascending' | 'descending';
+    sort?: ISortMode;
 }
 
 export interface DataSet {
@@ -157,6 +167,11 @@ export interface IFilterField extends IViewField {
     rule: IFilterRule | null;
 }
 
+export interface IFilterFiledSimple {
+    fid: string;
+    rule: IFilterRule | null;
+}
+
 export interface DraggableFieldState {
     dimensions: IViewField[];
     measures: IViewField[];
@@ -192,7 +207,7 @@ export type IFilterRule =
           value: Set<string | number>;
       };
 
-export type IStackMode = 'none' | 'stack' | 'normalize';
+export type IStackMode = 'none' | 'stack' | 'normalize' | 'zero' | 'center';
 
 export interface IVisualConfig {
     defaultAggregated: boolean;
@@ -200,18 +215,28 @@ export interface IVisualConfig {
     stack: IStackMode;
     showActions: boolean;
     interactiveScale: boolean;
-    sorted: 'none' | 'ascending' | 'descending';
+    sorted: ISortMode;
     zeroScale: boolean;
+    background?: string;
     format: {
         numberFormat?: string;
         timeFormat?: string;
         normalizedNumberFormat?: string;
     };
+    resolve: {
+        x?: boolean;
+        y?: boolean;
+        color?: boolean;
+        opacity?: boolean;
+        shape?: boolean;
+        size?: boolean;
+    };
     size: {
         mode: 'auto' | 'fixed';
         width: number;
         height: number;
     };
+    limit: number;
 }
 
 export interface IVisSpec {
@@ -238,6 +263,7 @@ export enum ISegmentKey {
 
 export type IThemeKey = 'vega' | 'g2';
 export type IDarkMode = 'media' | 'light' | 'dark';
+export type IComputationFunction = (payload: IDataQueryPayload) => Promise<IRow[]>;
 
 export type VegaGlobalConfig = VgConfig | VlConfig;
 
@@ -345,3 +371,80 @@ export interface IGWHandler {
 export interface IGWHandlerInsider extends IGWHandler {
     updateRenderStatus: (renderStatus: IRenderStatus) => void;
 }
+
+export interface IVisField {
+    key: string;
+    type: ISemanticType;
+    name?: string;
+    description?: string;
+    format?: string;
+    expression?: IExpression;
+}
+
+export type IVisFieldComputation = {
+    field: IVisField['key'];
+    expression: NonNullable<IVisField['expression']>;
+    name: NonNullable<IVisField['name']>;
+    type: IVisField['type'];
+};
+
+export interface IVisFilter {
+    fid: string;
+    rule: SetToArray<IFilterRule>;
+};
+
+export interface IFilterWorkflowStep {
+    type: 'filter';
+    filters: IVisFilter[];
+}
+
+export interface ITransformWorkflowStep {
+    type: 'transform';
+    transform: {
+        key: IVisFieldComputation['field'];
+        expression: IVisFieldComputation['expression'];
+    }[];
+}
+
+export interface IViewWorkflowStep {
+    type: 'view';
+    query: IViewQuery[];
+}
+
+export interface ISortWorkflowStep {
+    type: 'sort';
+    sort: 'ascending' | 'descending';
+    by: string[];
+}
+
+export type IDataQueryWorkflowStep = IFilterWorkflowStep | ITransformWorkflowStep | IViewWorkflowStep | ISortWorkflowStep;
+
+export interface IDataQueryPayload {
+    workflow: IDataQueryWorkflowStep[];
+    limit?: number;
+    offset?: number;
+}
+
+export interface ILoadDataPayload {
+    pageSize: number;
+    pageIndex: number;
+}
+
+export interface IGWDatasetStat {
+    count: number;
+}
+
+export type IResponse<T> = (
+    | {
+        success: true;
+        data: T;
+    }
+    | {
+        success: false;
+        message: string;
+        error?: {
+            code: `ERR_${Uppercase<string>}`;
+            options?: Record<string, string>;
+        };
+    }
+);

package/src/lib/execExp.ts

@@ -4,7 +4,7 @@ interface IDataFrame {
     [key: string]: any[];
 }
 
-export function execExpression (exp: IExpression, dataFrame: IDataFrame, columns: IField[]): IDataFrame {
+export function execExpression (exp: IExpression, dataFrame: IDataFrame): IDataFrame {
     const { op, params } = exp;
     const subFrame: IDataFrame = { ...dataFrame };
     const len = dataFrame[Object.keys(dataFrame)[0]].length;
@@ -17,7 +17,7 @@ export function execExpression (exp: IExpression, dataFrame: IDataFrame, columns
                 subFrame[param.value] = new Array(len).fill(param.value);
                 break;
             case 'expression':
-                let f = execExpression(param.value, dataFrame, columns);
+                let f = execExpression(param.value, dataFrame);
                 Object.keys(f).forEach(key => {
                     subFrame[key] = f[key];
                 })
@@ -54,11 +54,18 @@ function bin(resKey: string, params: IExpParamter[], data: IDataFrame, binSize:
         if (val < _min) _min = val;
     }
     const step = (_max - _min) / binSize;
-    const beaStep = Math.max(-Math.round(Math.log10(_max - _min)) + 2, 0)
+    // prevent (_max - _min) to be 0
+    const safeWidth = Math.min(Number.MAX_SAFE_INTEGER, Math.max(_max - _min, Number.MIN_VALUE));
+    const beaStep = Math.max(-Math.round(Math.log10(safeWidth)) + 2, 0)
+    // toFix() accepts 0-100
+    const safeBeaStep = Math.min(100, Math.max(0, Math.max(Number.isFinite(beaStep) ? beaStep : 0, 0)));
     const newValues = fieldValues.map((v: number) => {
         let bIndex = Math.floor((v - _min) / step);
         if (bIndex === binSize) bIndex = binSize - 1;
-        return Number(((bIndex * step + _min)).toFixed(beaStep))
+        if (Number.isNaN(bIndex)) {
+            bIndex = 0;
+        }
+        return Number(((bIndex * step + _min)).toFixed(safeBeaStep))
     });
     return {
         ...data,
@@ -124,22 +131,24 @@ function one(resKey: string, params: IExpParamter[], data: IDataFrame): IDataFra
     }
 }
 
-export function dataset2DataFrame(dataset: IRow[], columns: IField[]): IDataFrame {
+export function dataset2DataFrame(dataset: IRow[]): IDataFrame {
     const dataFrame: IDataFrame = {};
-    columns.forEach((col) => {
-        dataFrame[col.fid] = dataset.map((row) => row[col.fid]);
+    if (dataset.length === 0) return dataFrame;
+    Object.keys(dataset[0]).forEach((k) => {
+        dataFrame[k] = dataset.map((row) => row[k]);
     });
     return dataFrame;
 }
 
-export function dataframe2Dataset(dataFrame: IDataFrame, columns: IField[]): IRow[] {
-    if (columns.length === 0) return [];
+export function dataframe2Dataset(dataFrame: IDataFrame): IRow[] {
+    const cols = Object.keys(dataFrame);
+    if (cols.length === 0) return [];
     const dataset: IRow[] = [];
     const len = dataFrame[Object.keys(dataFrame)[0]].length;
     for (let i = 0; i < len; i++) {
         const row: IRow = {};
-        columns.forEach((col) => {
-            row[col.fid] = dataFrame[col.fid][i];
+        cols.forEach((k) => {
+            row[k] = dataFrame[k][i];
         });
         dataset.push(row);
     }