@kanaries/graphic-walker 0.2.3 → 0.2.5

package/src/utils/index.ts

@@ -0,0 +1,251 @@
+import i18next from 'i18next';
+import { COUNT_FIELD_ID } from '../constants';
+import { IRow, Filters, IMutField } from '../interfaces';
+interface NRReturns {
+    normalizedData: IRow[];
+    maxMeasures:IRow;
+    minMeasures:IRow;
+    totalMeasures:IRow
+}
+function normalizeRecords(dataSource: IRow[], measures: string[]): NRReturns {
+    const maxMeasures: IRow = {};
+    const minMeasures: IRow = {};
+    const totalMeasures: IRow = {};
+    measures.forEach(mea => {
+        maxMeasures[mea] = -Infinity;
+        minMeasures[mea] = Infinity;
+        totalMeasures[mea] = 0;
+    })
+    dataSource.forEach(record => {
+        measures.forEach(mea => {
+            maxMeasures[mea] = Math.max(record[mea], maxMeasures[mea])
+            minMeasures[mea] = Math.min(record[mea], minMeasures[mea])
+        })
+    })
+    const newData: IRow[] = [];
+    dataSource.forEach(record => {
+        const norRecord: IRow = { ... record };
+        measures.forEach(mea => {
+            // norRecord[mea] = norRecord[mea] - minMeasures[mea]
+            totalMeasures[mea] += Math.abs(norRecord[mea]);
+        })
+        newData.push(norRecord)
+    })
+    newData.forEach(record => {
+        measures.forEach(mea => {
+            record[mea] /= totalMeasures[mea];
+        })
+    })
+    return {
+        normalizedData: newData,
+        maxMeasures,
+        minMeasures,
+        totalMeasures
+    }
+}
+
+function normalize2PositiveRecords(dataSource: IRow[], measures: string[]): NRReturns {
+  const maxMeasures: IRow = {};
+  const minMeasures: IRow = {};
+  const totalMeasures: IRow = {};
+  measures.forEach((mea) => {
+    maxMeasures[mea] = -Infinity;
+    minMeasures[mea] = Infinity;
+    totalMeasures[mea] = 0;
+  });
+  dataSource.forEach((record) => {
+    measures.forEach((mea) => {
+      maxMeasures[mea] = Math.max(record[mea], maxMeasures[mea]);
+      minMeasures[mea] = Math.min(record[mea], minMeasures[mea]);
+    });
+  });
+  const newData: IRow[] = [];
+  dataSource.forEach((record) => {
+    const norRecord: IRow = { ...record };
+    measures.forEach((mea) => {
+      norRecord[mea] = norRecord[mea] - minMeasures[mea]
+      totalMeasures[mea] += norRecord[mea];
+    });
+    newData.push(norRecord);
+  });
+  newData.forEach((record) => {
+    measures.forEach((mea) => {
+      record[mea] /= totalMeasures[mea];
+    //   if (isNaN(record[mea])) {
+    //       record[mea] = 1
+    //   }
+    });
+  });
+  return {
+    normalizedData: newData,
+    maxMeasures,
+    minMeasures,
+    totalMeasures,
+  };
+}
+
+export function checkMajorFactor(data: IRow[], childrenData: Map<any, IRow[]>, dimensions: string[], measures: string[]): { majorKey: string; majorSum: number } {
+    const { normalizedData, maxMeasures, minMeasures, totalMeasures } = normalizeRecords(data, measures);
+    let majorSum = Infinity;
+    let majorKey = '';
+    for (let [key, childData] of childrenData) {
+        let sum = 0;
+        for (let record of normalizedData ) {
+            let target = childData.find(childRecord => {
+                return dimensions.every(dim => record[dim] === childRecord[dim])
+            })
+            if (target) {
+                measures.forEach(mea => {
+                    let targetValue = (typeof target![mea] === 'number' && !isNaN(target![mea])) ? target![mea] : 0;
+                    targetValue = (targetValue) / totalMeasures[mea]
+                    sum += Math.abs(record[mea] - targetValue)
+                })
+            } else {
+                measures.forEach(mea => {
+                    sum += Math.abs(record[mea]);
+                })
+            }
+        }
+        if (sum < majorSum) {
+            majorSum = sum;
+            majorKey = key;
+        }
+    }
+    majorSum /= (measures.length * 2);
+    return { majorKey, majorSum };
+}
+
+export function checkChildOutlier(data: IRow[], childrenData: Map<any, IRow[]>, dimensions: string[], measures: string[]): { outlierKey: string; outlierSum: number } {
+    // const { normalizedData, maxMeasures, minMeasures, totalMeasures } = normalize2PositiveRecords(data, measures);
+    const { normalizedData, maxMeasures, minMeasures, totalMeasures } = normalizeRecords(data, measures);
+    let outlierSum = -Infinity;
+    let outlierKey = '';
+    for (let [key, childData] of childrenData) {
+        // const { normalizedData: normalizedChildData } = normalize2PositiveRecords(childData, measures);
+        const { normalizedData: normalizedChildData } = normalizeRecords(childData, measures);
+        let sum = 0;
+        for (let record of normalizedData ) {
+            let target = normalizedChildData.find(childRecord => {
+                return dimensions.every(dim => record[dim] === childRecord[dim])
+            })
+            if (target) {
+                measures.forEach(mea => {
+                    let targetValue = (typeof target![mea] === 'number' && !isNaN(target![mea])) ? target![mea] : 0;
+                    sum += Math.abs(record[mea] - targetValue)
+                })
+            } else {
+                measures.forEach(mea => {
+                    sum += Math.abs(record[mea]);
+                })
+            }
+        }
+        if (sum > outlierSum) {
+            outlierSum = sum;
+            outlierKey = key;
+        }
+    }
+    outlierSum /= (measures.length * 2);
+    return { outlierKey, outlierSum };
+}
+export interface IPredicate {
+    key: string;
+    type: 'discrete' | 'continuous';
+    range: Set<any> | [number, number];
+}
+export function getPredicates(selection: IRow[], dimensions: string[], measures: string[]): IPredicate[] {
+    const predicates: IPredicate[] = [];
+    dimensions.forEach(dim => {
+        predicates.push({
+            key: dim,
+            type: 'discrete',
+            range: new Set()
+        })
+    })
+    measures.forEach(mea => {
+        predicates.push({
+            key: mea,
+            type: 'continuous',
+            range: [Infinity, -Infinity]
+        })
+    })
+    selection.forEach(record => {
+        dimensions.forEach((dim, index) => {
+            (predicates[index].range as Set<any>).add(record[dim])
+        })
+        measures.forEach((mea, index) => {
+            (predicates[index].range as [number, number])[0] = Math.min(
+              (predicates[index].range as [number, number])[0],
+              record[mea]
+            );
+            (predicates[index].range as [number, number])[1] = Math.max(
+              (predicates[index].range as [number, number])[1],
+              record[mea]
+            );
+        })
+    })
+    return predicates;
+}
+
+export function getPredicatesFromVegaSignals(signals: Filters, dimensions: string[], measures: string[]): IPredicate[] {
+    const predicates: IPredicate[] = [];
+    dimensions.forEach(dim => {
+        predicates.push({
+            type: 'discrete',
+            range: new Set(signals[dim]),
+            key: dim
+        });
+    });
+    return predicates;
+}
+
+export function filterByPredicates(data: IRow[], predicates: IPredicate[]): IRow[] {
+    const filterData = data.filter((record) => {
+      return predicates.every((pre) => {
+        if (pre.type === 'continuous') {
+          return (
+            record[pre.key] >= (pre.range as [number, number])[0] &&
+            record[pre.key] <= (pre.range as [number, number])[1]
+          );
+        } else {
+          return (pre.range as Set<any>).has(record[pre.key]);
+        }
+      });
+    });
+    return filterData;
+}
+
+export function applyFilters(dataSource: IRow[], filters: Filters): IRow[] {
+    let filterKeys = Object.keys(filters);
+    return dataSource.filter((record) => {
+        let keep = true;
+        for (let filterKey of filterKeys) {
+            if (filters[filterKey].length > 0) {
+                if (!filters[filterKey].includes(record[filterKey])) {
+                    keep = false;
+                    break;
+                }
+            }
+        }
+        return keep;
+    });
+}
+
+export function extendCountField (dataSource: IRow[], fields: IMutField[]): {
+    dataSource: IRow[];
+    fields: IMutField[];
+} {
+    const nextData = dataSource.map(r => ({
+        ...r,
+        [COUNT_FIELD_ID]: 1
+    }))
+    const nextFields = fields.concat({
+        fid: COUNT_FIELD_ID,
+        name: i18next.t('constant.row_count'),
+        analyticType: 'measure',
+        semanticType: 'quantitative'
+    })
+    return {
+        dataSource: nextData,
+        fields: nextFields
+    }
+}

package/src/utils/normalization.ts

@@ -0,0 +1,158 @@
+import { IRow } from '../interfaces';
+
+export function normalizeWithParent(
+    data: IRow[],
+    parentData: IRow[],
+    measures: string[],
+    syncScale: boolean
+): {
+    normalizedData: IRow[];
+    normalizedParentData: IRow[];
+} {
+    const totalMeasuresOfParent: IRow = {};
+    const totalMeasures: IRow = {};
+    measures.forEach(mea => {
+        totalMeasuresOfParent[mea] = 0;
+        totalMeasures[mea] = 0;
+    })
+    parentData.forEach(record => {
+        measures.forEach(mea => {
+            totalMeasuresOfParent[mea] += Math.abs(record[mea])
+        })
+    })
+    data.forEach(record => {
+        measures.forEach(mea => {
+            totalMeasures[mea] += Math.abs(record[mea]);
+        })
+    })
+    const normalizedParentData: IRow[] = [];
+    parentData.forEach(record => {
+        const newRecord = { ...record };
+        measures.forEach(mea => {
+            newRecord[mea] /= totalMeasuresOfParent[mea];
+        })
+        normalizedParentData.push(newRecord);
+    })
+    const normalizedData: IRow[] = [];
+    data.forEach(record => {
+        const newRecord = { ...record };
+        measures.forEach(mea => {
+            if (syncScale) {
+                newRecord[mea] /= totalMeasuresOfParent[mea];
+            } else {
+                newRecord[mea] /= totalMeasures[mea]
+            }
+        })
+        normalizedData.push(newRecord);
+    })
+    return {
+        normalizedData,
+        normalizedParentData
+    };
+}
+
+export function compareDistribution (distribution1: IRow[], distribution2: IRow[], dimensions: string[], measures: string[]): number {
+    let score = 0;
+    let count = 0;
+    const tagsForD2: boolean[] = distribution2.map(() => false);
+    for (let record of distribution1) {
+        let targetRecordIndex = distribution2.findIndex((r, i) => {
+            return !tagsForD2[i] && dimensions.every(dim => r[dim] === record[dim])
+        })
+        if (targetRecordIndex > -1) {
+            tagsForD2[targetRecordIndex] = true;
+            const targetRecord = distribution2[targetRecordIndex];
+            for (let mea of measures) {
+                // score += Math.abs(targetRecord[mea] - record[mea]);
+                // if (targetRecord[mea] === 0 || record[mea] === 0) continue;
+                // score += Math.max(targetRecord[mea], record[mea]) / Math.min(targetRecord[mea], record[mea]);
+                
+                score = Math.max(
+                    score,
+                    Math.max(targetRecord[mea], record[mea]) /
+                        Math.min(targetRecord[mea], record[mea])
+                );
+                count++;
+            }
+        } else {
+            for (let mea of measures) {
+                score = Math.max(score, record[mea])
+                // score += Math.abs(record[mea])
+                count++;
+            }
+        }
+    }
+    for (let i = 0; i < distribution2.length; i++) {
+        if (!tagsForD2[i]) {
+            tagsForD2[i] = true;
+            for (let mea of measures) {
+                // score += Math.abs(distribution2[i][mea])
+                score = Math.max(score, distribution2[i][mea]);
+                count++;
+            }
+        }
+    }
+    return score;
+}
+
+export function normalizeByMeasures (dataSource: IRow[], measures: string[]) {
+    let sums: Map<string, number> = new Map();
+
+    measures.forEach(mea => {
+        sums.set(mea, 0);
+    })
+
+    dataSource.forEach(record => {
+        measures.forEach(mea => {
+            sums.set(mea, sums.get(mea)! + Math.abs(record[mea]));
+        })
+    })
+
+    const ans: IRow[] = [];
+    dataSource.forEach(record => {
+        const norRecord: IRow = { ...record };
+        measures.forEach(mea => {
+            norRecord[mea] /= sums.get(mea)!;
+        })
+        ans.push(norRecord);
+    });
+    return ans;
+}
+
+export function getDistributionDifference(dataSource: IRow[], dimensions: string[], measure1: string, measure2: string): number {
+    let score = 0;
+    for (let record of dataSource) {
+        // score += Math.abs(record[measure1] - record[measure2])
+        if (record[measure1] === 0 || record[measure2] === 0) continue;
+        score += Math.max(record[measure1], record[measure2]) / Math.min(record[measure1], record[measure2]);
+    }
+    return score;
+}
+
+export function makeBinField (dataSource: IRow[], fid: string, binFid: string, binSize: number | undefined = 10) {
+    let _min = Infinity;
+    let _max = -Infinity;
+    for (let i = 0; i < dataSource.length; i++) {
+        let val = dataSource[i][fid];
+        if (val > _max) _max = val;
+        if (val < _min) _min = val;
+    }
+    const step = (_max - _min) / binSize;
+    return dataSource.map(r => {
+        let bIndex = Math.floor((r[fid] - _min) / step);
+        if (bIndex === binSize) bIndex = binSize - 1;
+        return {
+            ...r,
+            [binFid]: bIndex * step + _min
+        }
+    })
+}
+
+export function makeLogField (dataSource: IRow[], fid: string, logFid: string) {
+    return dataSource.map(r => {
+        return {
+            ...r,
+            [logFid]: (typeof r[fid] === 'number' && r[fid] > 0) ? Math.log10(r[fid]) : null
+        }
+    })
+}

package/src/utils/save.ts

@@ -0,0 +1,46 @@
+import { IDataSet, IDataSource, IVisSpec } from "../interfaces";
+import { VisSpecWithHistory } from "../models/visSpecHistory";
+
+
+export function dumpsGWPureSpec (list: VisSpecWithHistory[]): IVisSpec[] {
+    return list.map(l => l.exportGW())
+}
+
+export function parseGWPureSpec (list: IVisSpec[]): VisSpecWithHistory[] {
+    return list.map(l => new VisSpecWithHistory(l))
+}
+
+interface IStoInfo {
+    datasets: IDataSet[];
+    specList: IVisSpec[];
+    dataSources: IDataSource[]
+}
+
+export function stringifyGWContent (info: IStoInfo) {
+    return JSON.stringify(info)
+}
+
+export function parseGWContent (raw: string): IStoInfo {
+    return JSON.parse(raw)
+}
+// JSON.stringify
+
+export function download(data: string, filename: string, type: string) {
+    var file = new Blob([data], {type: type});
+    // @ts-ignore
+    if (window.navigator.msSaveOrOpenBlob) // IE10+
+        // @ts-ignore
+        window.navigator.msSaveOrOpenBlob(file, filename);
+    else { // Others
+        var a = document.createElement("a"),
+                url = URL.createObjectURL(file);
+        a.href = url;
+        a.download = filename;
+        document.body.appendChild(a);
+        a.click();
+        setTimeout(function() {
+            document.body.removeChild(a);
+            window.URL.revokeObjectURL(url);  
+        }, 0); 
+    }
+}

package/src/vis/future-react-vega.tsx

@@ -0,0 +1,193 @@
+/**
+ * TODO: This file will be used when vega-lite facets bug is fixed.
+ * https://github.com/vega/vega-lite/issues/4680
+ */
+import React, { useEffect, useRef } from 'react';
+import embed from 'vega-embed';
+import { Subject } from 'rxjs'
+import * as op from 'rxjs/operators';
+import { ScenegraphEvent } from 'vega';
+import { IField, IRow } from '../interfaces';
+
+const SELECTION_NAME = 'geom';
+interface ReactVegaProps {
+    rows: IField[];
+    columns: IField[];
+    dataSource: IRow[];
+    defaultAggregate?: boolean;
+    geomType: string;
+    color?: IField;
+    opacity?: IField;
+    size?: IField;
+    onGeomClick?: (values: any, e: any) => void
+}
+const NULL_FIELD: IField = {
+    fid: '',
+    name: '',
+    semanticType: 'quantitative',
+    analyticType: 'measure',
+    aggName: 'sum'
+}
+const click$ = new Subject<ScenegraphEvent>();
+const selection$ = new Subject<any>();
+const geomClick$ = selection$.pipe(
+    op.withLatestFrom(click$),
+    op.filter(([values, _]) => {
+        if (Object.keys(values).length > 0) {
+            return true
+        }
+        return false
+    })
+);
+function getFieldType(field: IField): 'quantitative' | 'nominal' | 'ordinal' | 'temporal' {
+    if (field.analyticType === 'measure') return 'quantitative';
+    return 'nominal';
+}
+
+function getSingleView(xField: IField, yField: IField, color: IField, opacity: IField, size: IField, row: IField, col: IField, defaultAggregated: boolean, geomType: string) {
+    return {
+        mark: geomType,
+        encoding: {
+            x: {
+                field: xField.fid,
+                type: getFieldType(xField),
+                aggregate:
+                    xField.analyticType === 'measure' &&
+                    defaultAggregated &&
+                    (xField.aggName as any),
+            },
+            y: {
+                field: yField.fid,
+                type: getFieldType(yField),
+                aggregate:
+                    yField.analyticType === 'measure' &&
+                    defaultAggregated &&
+                    (yField.aggName as any),
+            },
+            row: row !== NULL_FIELD ? {
+                field: row.fid,
+                type: getFieldType(row),
+            } : undefined,
+            column: col !== NULL_FIELD ? {
+                field: col.fid,
+                type: getFieldType(col),
+            } : undefined,
+            color: color !== NULL_FIELD ? {
+                field: color.fid,
+                type: getFieldType(color)
+            } : undefined,
+            opacity: opacity !== NULL_FIELD ? {
+                field: opacity.fid,
+                type: getFieldType(opacity)
+            } : undefined,
+            size: size !== NULL_FIELD ? {
+                field: size.fid,
+                type: getFieldType(size)
+            } : undefined
+        }
+    };
+}
+const ReactVega: React.FC<ReactVegaProps> = props => {
+    const {
+        dataSource = [],
+        rows = [],
+        columns = [],
+        defaultAggregate = true,
+        geomType,
+        color,
+        opacity,
+        size,
+        onGeomClick
+    } = props;
+    const container = useRef<HTMLDivElement>(null);
+    useEffect(() => {
+        const clickSub = geomClick$.subscribe(([values, e]) => {
+            if (onGeomClick) {
+                onGeomClick(values, e);
+            }
+        })
+        return () => {
+            clickSub.unsubscribe();
+        }
+    }, []);
+    useEffect(() => {
+        if (container.current) {
+            const rowDims = rows.filter(f => f.analyticType === 'dimension');
+            const colDims = columns.filter(f => f.analyticType === 'dimension');
+            const rowMeas = rows.filter(f => f.analyticType === 'measure');
+            const colMeas = columns.filter(f => f.analyticType === 'measure');
+
+            const yField = rows.length > 0 ? rows[rows.length - 1] : NULL_FIELD;
+            const xField = columns.length > 0 ? columns[columns.length - 1] : NULL_FIELD;
+
+            const rowFacetFields = rowDims.slice(0, -1);
+            const colFacetFields = colDims.slice(0, -1);
+            const rowFacetField = rowFacetFields.length > 0 ? rowFacetFields[rowFacetFields.length - 1] : NULL_FIELD;
+            const colFacetField = colFacetFields.length > 0 ? colFacetFields[colFacetFields.length - 1] : NULL_FIELD;
+
+            const rowRepeatFields = rowMeas.length === 0 ? rowDims.slice(-1) : rowMeas;//rowMeas.slice(0, -1);
+            const colRepeatFields = colMeas.length === 0 ? colDims.slice(-1) : colMeas;//colMeas.slice(0, -1);
+
+            const rowRepeatField = rowRepeatFields.length > 0 ? rowRepeatFields[rowRepeatFields.length - 1] : NULL_FIELD;
+            const colRepeatField = colRepeatFields.length > 0 ? colRepeatFields[colRepeatFields.length - 1] : NULL_FIELD;
+
+            const dimensions = [...rows, ...columns, color, opacity, size].filter(f => Boolean(f)).map(f => (f as IField).fid)
+
+            const spec: any = {
+                data: {
+                    values: dataSource,
+                },
+                selection: {
+                    [SELECTION_NAME]: {
+                        type: 'single',
+                        fields: dimensions
+                    }
+                }
+            };
+            if (false) {
+                // const singleView = getSingleView(
+                //   xField,
+                //   yField,
+                //   color ? color : NULL_FIELD,
+                //   opacity ? opacity : NULL_FIELD,
+                //   size ? size : NULL_FIELD,
+                //   rowFacetField,
+                //   colFacetField,
+                //   defaultAggregate,
+                //   geomType
+                // );
+                // spec.mark = singleView.mark;
+                // spec.encoding = singleView.encoding;
+            } else {
+                spec.concat = [];
+                for (let i = 0; i < rowRepeatFields.length; i++) {
+                    for (let j = 0; j < colRepeatFields.length; j++) {
+                        const singleView = getSingleView(
+                            colRepeatFields[j] || NULL_FIELD,
+                            rowRepeatFields[i] || NULL_FIELD,
+                            color ? color : NULL_FIELD,
+                            opacity ? opacity : NULL_FIELD,
+                            size ? size : NULL_FIELD,
+                            rowFacetField,
+                            colFacetField,
+                            defaultAggregate,
+                            geomType
+                        );
+                        spec.concat.push(singleView)
+                    }
+                }
+            }
+            embed(container.current, spec, { mode: 'vega-lite', actions: false }).then(res => {
+                res.view.addEventListener('click', (e) => {
+                    click$.next(e);
+                })
+                res.view.addSignalListener(SELECTION_NAME, (name: any, values: any) => {
+                    selection$.next(values);
+                });
+            });
+        }
+    }, [dataSource, rows, columns, defaultAggregate, geomType, color, opacity, size]);
+    return <div ref={container}></div>
+}
+
+export default ReactVega;

package/src/vis/gen-vega.tsx

@@ -0,0 +1,52 @@
+import React, { useRef, useEffect, useState } from 'react';
+import embed, { vega, Result } from 'vega-embed';
+import { Spec } from 'vega';
+
+interface GenVegaProps {
+  dataSource: any[];
+  spec: Spec;
+  signalHandler?: {
+    [key: string]: (name: any, value: any) => void;
+  };
+}
+const GenVega: React.FC<GenVegaProps> = (props) => {
+  const { spec, dataSource, signalHandler = {} } = props;
+  const container = useRef<HTMLDivElement>(null);
+  const [view, setView] = useState<Result['view']>();
+  useEffect(() => {
+    if (container.current) {
+      embed(container.current, spec).then((res) => {
+        setView(res.view);
+      });
+    }
+  }, [spec]);
+  useEffect(() => {
+    if (view && signalHandler) {
+      for (let key in signalHandler) {
+        view.addSignalListener('sl', signalHandler[key]);
+      }
+    }
+    return () => {
+      if (view && signalHandler) {
+        for (let key in signalHandler) {
+          view.removeSignalListener('sl', signalHandler[key]);
+        }
+      }
+    };
+  }, [view, signalHandler]);
+  useEffect(() => {
+    view &&
+      view.change(
+        'dataSource',
+        vega
+          .changeset()
+          .remove(() => true)
+          .insert(dataSource)
+      );
+    view && view.resize();
+    view && view.runAsync();
+  }, [view, dataSource]);
+  return <div ref={container} />;
+};
+
+export default GenVega;