buckaroo-js-core 0.8.0

package/src/buckaroo_js/components/DFViewerParts/gridUtils.ts

@@ -0,0 +1,290 @@
+import {
+    CellClassParams,
+    CellRendererSelectorResult,
+    ColDef,
+    ICellRendererParams
+} from 'ag-grid-community';
+import {BLUE_TO_YELLOW, DIVERGING_RED_WHITE_BLUE} from '../../baked_data/colorMap';
+
+import {
+    DFWhole,
+    DisplayerArgs,
+    cellRendererDisplayers,
+    ColumnConfig,
+    ColorMappingConfig,
+    ColorMapRules,
+    TooltipConfig,
+    ColorWhenNotNullRules,
+    DFViewerConfig
+} from './DFWhole';
+import _, {zipObject} from 'lodash';
+import {getTextCellRenderer} from './HistogramCell';
+
+import {DFData, SDFMeasure, SDFT} from './DFWhole';
+
+import {CellRendererArgs, FormatterArgs, PinnedRowConfig} from './DFWhole';
+import {getBakedDFViewer} from './SeriesSummaryTooltip';
+import {getFormatterFromArgs, getCellRenderer, objFormatter, getFormatter} from './Displayer';
+
+// for now colDef stuff with less than 3 implementantions should stay in this file
+// as implementations grow large or with many implmentations, they should move to separate files
+// like displayer
+
+export function addToColDef(dispArgs: DisplayerArgs, summary_stats_column: SDFMeasure) {
+    const formatter = getFormatterFromArgs(dispArgs);
+    if (formatter !== undefined) {
+        const colDefExtras: ColDef = {valueFormatter: formatter};
+        return colDefExtras;
+    }
+
+    if (_.includes(cellRendererDisplayers, dispArgs.displayer)) {
+        const crArgs: CellRendererArgs = dispArgs as CellRendererArgs;
+        return {
+            cellRenderer: getCellRenderer(crArgs)
+        };
+    }
+    return undefined;
+}
+
+export function getHistoIndex(val: number, histogram_edges: number[]): number {
+    /*
+np.histogram([1, 2, 3, 4,  10, 20, 30, 40, 300, 300, 400, 500], bins=5)
+( [  8,       0,     2,     1,     1], 
+[  1. , 100.8, 200.6, 300.4, 400.2, 500. ])
+The bottom matters for us, those are the endge
+
+this means that 8 values are between 1 and 100.8  and 2 values are between 200.6 and 300.4
+  */
+    if (histogram_edges.length === 0) {
+        return 0;
+    }
+    for (let i = 0; i < histogram_edges.length; i++) {
+        if (val <= histogram_edges[i]) {
+            return i;
+        }
+    }
+    return histogram_edges.length;
+}
+
+export function colorMap(cmr: ColorMapRules, histogram_edges: number[]) {
+    // https://colorcet.com/gallery.html#isoluminant
+    // https://github.com/holoviz/colorcet/tree/main/assets/CET
+    // https://github.com/bokeh/bokeh/blob/ed285b11ab196e72336b47bf12f44e1bef5abed3/src/bokeh/models/mappers.py#L304
+    const maps: Record<string, string[]> = {
+        BLUE_TO_YELLOW: BLUE_TO_YELLOW,
+        DIVERGING_RED_WHITE_BLUE: DIVERGING_RED_WHITE_BLUE
+    };
+    const cmap = maps[cmr.map_name];
+
+    function numberToColor(val: number) {
+        const histoIndex = getHistoIndex(val, histogram_edges);
+        const scaledIndex = Math.round((histoIndex / histogram_edges.length) * cmap.length);
+        return cmap[scaledIndex];
+    }
+
+    function cellStyle(params: CellClassParams) {
+        const val = cmr.val_column ? params.data[cmr.val_column] : params.value;
+        const color = numberToColor(val);
+        return {
+            backgroundColor: color
+        };
+    }
+
+    const retProps = {
+        cellStyle: cellStyle
+    };
+    return retProps;
+}
+
+export function colorNotNull(cmr: ColorWhenNotNullRules) {
+    function cellStyle(params: CellClassParams) {
+        if (params.data === undefined) {
+            return {backgroundColor: 'inherit'};
+        }
+        const val = params.data[cmr.exist_column];
+        const valPresent = val && val !== null;
+        const isPinned = params.node.rowPinned;
+        const color = valPresent && !isPinned ? cmr.conditional_color : 'inherit';
+        return {
+            backgroundColor: color
+        };
+    }
+
+    const retProps = {
+        cellStyle: cellStyle
+    };
+    return retProps;
+}
+
+export function getStyler(cmr: ColorMappingConfig, col_name: string, histogram_stats: SDFT) {
+    switch (cmr.color_rule) {
+        case 'color_map': {
+            //block necessary because you cant define varaibles in case blocks
+            const statsCol = cmr.val_column || col_name;
+            const summary_stats_cell = histogram_stats[statsCol];
+
+            if (summary_stats_cell && summary_stats_cell.histogram_bins !== undefined) {
+                return colorMap(cmr, summary_stats_cell.histogram_bins);
+            } else {
+                console.log('histogram bins not found for color_map');
+                return {};
+            }
+        }
+        case 'color_not_null':
+            return colorNotNull(cmr);
+    }
+}
+
+export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[]) {
+    return _.map(_.map(prc, 'primary_key_val'), (x) => _.find(sdf, {index: x}));
+}
+
+export function getTooltip(ttc: TooltipConfig, single_series_summary_df: DFWhole): Partial<ColDef> {
+    switch (ttc.tooltip_type) {
+        case 'simple':
+            return {tooltipField: ttc.val_column};
+
+        case 'summary_series':
+            return {
+                tooltipComponent: getBakedDFViewer(single_series_summary_df),
+                tooltipField: 'index',
+                tooltipComponentParams: {}
+            };
+    }
+}
+
+export function extractSingleSeriesSummary(
+    full_summary_stats_df: DFData,
+    col_name: string
+): DFWhole {
+    return {
+        dfviewer_config: {
+            column_config: [
+                {col_name: 'index', displayer_args: {displayer: 'obj'}},
+                {col_name: col_name, displayer_args: {displayer: 'obj'}}
+            ],
+            pinned_rows: []
+        },
+        data: _.filter(
+            _.map(full_summary_stats_df, (row) => _.pick(row, ['index', col_name])),
+            {index: 'dtype'}
+        )
+    };
+}
+
+export function dfToAgrid(
+    tdf: DFData,
+    dfviewer_config: DFViewerConfig,
+    full_summary_stats_df: DFData
+): [ColDef[], unknown[]] {
+    //more convienient df format for some formatters
+    const hdf = extractSDFT(full_summary_stats_df || []);
+
+    const retColumns: ColDef[] = dfviewer_config.column_config.map((f: ColumnConfig) => {
+        const single_series_summary_df = extractSingleSeriesSummary(
+            full_summary_stats_df,
+            f.col_name
+        );
+
+        const color_map_config = f.color_map_config
+            ? getStyler(f.color_map_config, f.col_name, hdf)
+            : {};
+
+        const tooltip_config = f.tooltip_config
+            ? getTooltip(f.tooltip_config, single_series_summary_df)
+            : {};
+        const colDef: ColDef = {
+            field: f.col_name,
+            headerName: f.col_name,
+            cellDataType: false,
+            cellStyle: {}, // necessary for colormapped columns to have a default
+            ...addToColDef(f.displayer_args, hdf[f.col_name]),
+            ...color_map_config,
+            ...tooltip_config,
+            ...f.ag_grid_specs
+        };
+        return colDef;
+    });
+    return [retColumns, tdf];
+}
+
+// this is very similar to the colDef parts of dfToAgrid
+export function getCellRendererSelector(pinned_rows: PinnedRowConfig[]) {
+    const anyRenderer: CellRendererSelectorResult = {
+        component: getTextCellRenderer(objFormatter)
+    };
+    return (params: ICellRendererParams<any, any, any>): CellRendererSelectorResult | undefined => {
+        if (params.node.rowPinned) {
+            const pk = _.get(params.node.data, 'index');
+            if (pk === undefined) {
+                return anyRenderer; // default renderer
+            }
+            const maybePrc: PinnedRowConfig | undefined = _.find(pinned_rows, {
+                primary_key_val: pk
+            });
+            if (maybePrc === undefined) {
+                return anyRenderer;
+            }
+            const prc: PinnedRowConfig = maybePrc;
+            console.log('params', params);
+            const currentCol = params.column?.getColId();
+            if (
+                (prc.default_renderer_columns === undefined && currentCol === 'index') ||
+                _.includes(prc.default_renderer_columns, currentCol)
+            ) {
+                return anyRenderer;
+            }
+            const possibCellRenderer = getCellRenderer(prc.displayer_args as CellRendererArgs);
+
+            if (possibCellRenderer === undefined) {
+                const formattedRenderer: CellRendererSelectorResult = {
+                    component: getTextCellRenderer(
+                        getFormatter(prc.displayer_args as FormatterArgs)
+                    )
+                };
+                return formattedRenderer;
+            }
+            return {component: possibCellRenderer};
+        } else {
+            return undefined; // rows that are not pinned don't use a row level cell renderer
+        }
+    };
+}
+
+export function extractSDFT(summaryStatsDf: DFData): SDFT {
+    const maybeHistogramBins = _.find(summaryStatsDf, {index: 'histogram_bins'}) || {};
+    const maybeHistogramLogBins = _.find(summaryStatsDf, {index: 'histogram_log_bins'}) || {};
+    const allColumns: string[] = _.without(
+        _.union(_.keys(maybeHistogramBins), _.keys(maybeHistogramLogBins)),
+        'index'
+    );
+    const vals: SDFMeasure[] = _.map(allColumns, (colName) => {
+        return {
+            histogram_bins: _.get(maybeHistogramBins, colName, []) as number[],
+            histogram_log_bins: _.get(maybeHistogramLogBins, colName, []) as number[]
+        };
+    });
+    return zipObject(allColumns, vals) as SDFT;
+}
+
+/*
+I would love for extractSDF to be more elegant like the following function.  I just can't quite get it to work
+time to move on
+
+export function extractSDFT2(summaryStatsDf:DFData) : SDFT  {
+  const rows = ['histogram_bins', 'histogram_log_bins']
+
+  const extracted = _.map(rows, (pk) => {
+    return _.find(summaryStatsDf,  {'index': pk}) || {}
+  })
+  const dupKeys: string[][] = _.map(extracted, _.keys);
+  const allColumns: string[] = _.without(_.union(...dupKeys), 'index');
+  const vals:SDFMeasure[] = _.map(allColumns, (colName) => {
+    const pairs = _.map(_.zip(rows, extracted), (rname, row) => {
+      return [rname, (_.get(row, colName, []) as number[])];
+    })
+    return _.fromPairs(pairs) as SDFMeasure;
+  });
+  return zipObject(allColumns, vals) as SDFT;
+}
+*/

package/src/buckaroo_js/components/DependentTabs.tsx

@@ -0,0 +1,134 @@
+import React, {useState, CSSProperties, Dispatch, SetStateAction} from 'react';
+import {DFWhole, EmptyDf} from './DFViewerParts/DFWhole';
+import _ from 'lodash';
+import {Operation} from './OperationUtils';
+
+export function OperationDisplayer({
+    filledOperations,
+    style
+}: {
+    filledOperations: Operation[];
+    style: CSSProperties;
+}): React.JSX.Element {
+    const baseStyle: CSSProperties = {margin: '0', textAlign: 'left'};
+    const localStyle: CSSProperties = {...baseStyle, ...style};
+    return (
+        <div className='command-displayer' style={{width: '100%'}}>
+            <pre style={localStyle}>{JSON.stringify(filledOperations)}</pre>
+        </div>
+    );
+}
+
+export function PythonDisplayer({
+    style,
+    generatedPyCode
+}: {
+    style: CSSProperties;
+    generatedPyCode: string;
+}) {
+    const baseStyle: CSSProperties = {margin: '0', textAlign: 'left'};
+    const localStyle: CSSProperties = {...baseStyle, ...style};
+    return (
+        <div className='python-displayer' style={{width: '100%'}}>
+            <pre style={localStyle}>{generatedPyCode}</pre>
+        </div>
+    );
+}
+
+export function TransformViewer({
+    style,
+    transformedDf
+}: {
+    style: CSSProperties;
+    transformedDf: DFWhole;
+}) {
+    return <div className='transform-viewer'>"transformed view"</div>;
+}
+export type OperationResult = {
+    transformed_df: DFWhole;
+    generated_py_code: string;
+    transform_error?: string;
+};
+
+export type OrRequesterT = (ops: Operation[]) => void;
+export type getOperationResultSetterT = (
+    setter: Dispatch<SetStateAction<OperationResult>>
+) => OrRequesterT;
+
+export const baseOperationResults: OperationResult = {
+    transformed_df: EmptyDf,
+    generated_py_code: 'default py code'
+};
+
+export function TabComponent({
+    currentTab,
+    _setTab,
+    tabName
+}: {
+    currentTab: any;
+    _setTab: any;
+    tabName: any;
+}): JSX.Element {
+    return (
+        <li
+            onClick={() => {
+                _setTab(tabName);
+            }}
+            className={currentTab === tabName ? 'active' : ''}
+        >
+            {tabName}
+        </li>
+    );
+}
+
+export function DependentTabs({
+    filledOperations,
+    operationResult
+}: {
+    filledOperations: Operation[];
+    operationResult: OperationResult;
+}) {
+    const [tab, _setTab] = useState('DataFrame');
+    const style: CSSProperties = {height: '45vh'};
+    return (
+        <div className='dependent-tabs' style={{width: '100%'}}>
+            <ul className='tabs'>
+                <TabComponent currentTab={tab} _setTab={_setTab} tabName={'DataFrame'} />
+                <TabComponent currentTab={tab} _setTab={_setTab} tabName={'Python'} />
+                <TabComponent currentTab={tab} _setTab={_setTab} tabName={'Operations'} />
+            </ul>
+            <div className='output-area'>
+                {operationResult.transform_error ? (
+                    <div>
+                        <h2> error </h2>
+                        <PythonDisplayer
+                            style={style}
+                            generatedPyCode={operationResult.transform_error}
+                        />
+                    </div>
+                ) : (
+                    <span></span>
+                )}
+                {
+                    {
+                        Operations: (
+                            <OperationDisplayer style={style} filledOperations={filledOperations} />
+                        ),
+                        Python: (
+                            <PythonDisplayer
+                                style={style}
+                                generatedPyCode={operationResult.generated_py_code}
+                            />
+                        ),
+                        DataFrame: (
+                            <TransformViewer
+                                style={style}
+                                transformedDf={operationResult.transformed_df}
+                            />
+                        )
+                    }[tab]
+                }
+            </div>
+        </div>
+    );
+}

package/src/buckaroo_js/components/OperationDetail.tsx

@@ -0,0 +1,203 @@
+import _ from 'lodash';
+import {Operation, SettableArg, OperationEventFunc} from './OperationUtils';
+import {ActualArg, CommandArgSpec} from './CommandUtils';
+import {objWithoutNull, replaceAtIdx, replaceAtKey} from './utils';
+import React from 'react';
+
+export const OperationDetail = ({
+    command,
+    setCommand,
+    columns,
+    commandPatterns
+}: {
+    command: Operation;
+    setCommand: OperationEventFunc;
+    columns: string[];
+    commandPatterns: CommandArgSpec;
+}) => {
+    if (command === undefined) {
+        return <span></span>;
+    }
+    const commandName = command[0]['symbol'];
+    const pattern = commandPatterns[commandName];
+
+    if (!_.isArray(pattern)) {
+        //we shouldn't get here
+        return <h2>unknown command {commandName}</h2>;
+    } else if (_.isEqual(pattern, [null])) {
+        return <div className='operation-detail'></div>;
+    } else {
+        const fullPattern = pattern as ActualArg[];
+        return (
+            <div className='operation-detail'>
+                <ArgGetters
+                    command={command}
+                    fullPattern={fullPattern}
+                    setCommand={setCommand}
+                    columns={columns}
+                />
+            </div>
+        );
+    }
+    return <h2></h2>;
+};
+
+export const ArgGetters = ({
+    command,
+    fullPattern,
+    setCommand,
+    columns
+}: {
+    command: Operation;
+    fullPattern: ActualArg[];
+    setCommand: OperationEventFunc;
+    columns: string[];
+}) => {
+    /* reads the argspec and sets up the proper getters/setters */
+    const makeArgGetter = (pattern: ActualArg) => {
+        const idx = pattern[0];
+        const val = command[idx] as SettableArg;
+        const valSetter = (newVal: unknown) => {
+            const newCommand = replaceAtIdx(command, idx, newVal);
+            //console.log('newCommand', newCommand);
+            setCommand(newCommand as Operation);
+        };
+        return (
+            <div key={idx}>
+                <ArgGetter
+                    argProps={pattern}
+                    renderKey={idx}
+                    val={val}
+                    setter={valSetter}
+                    columns={columns}
+                />
+            </div>
+        );
+    };
+    return <div className='arg-getters'>{fullPattern.map(makeArgGetter)}</div>;
+};
+
+const ArgGetter = ({
+    argProps,
+    val,
+    setter,
+    columns,
+    renderKey
+}: {
+    argProps: ActualArg;
+    val: SettableArg;
+    setter: (arg: SettableArg) => void;
+    columns: string[];
+    renderKey: number;
+}) => {
+    const [_argPos, label, argType, lastArg] = argProps;
+
+    const defaultShim = (event: {target: {value: SettableArg}}) => setter(event.target.value);
+    if (argType === 'enum' && _.isArray(lastArg)) {
+        return (
+            <fieldset key={renderKey}>
+                <label> {label} </label>
+                <select defaultValue={val as string} onChange={defaultShim}>
+                    {lastArg.map((optionVal) => (
+                        <option key={optionVal} value={optionVal}>
+                            {optionVal}
+                        </option>
+                    ))}
+                </select>
+            </fieldset>
+        );
+    } else if (argType === 'type') {
+        if (lastArg === 'integer') {
+            const valSetterShim = (event: {target: {value: string}}) =>
+                setter(parseInt(event.target.value));
+            return (
+                <fieldset key={renderKey}>
+                    <label> {label} </label>
+                    <input
+                        type='number'
+                        defaultValue={val as number}
+                        step='1'
+                        onChange={valSetterShim}
+                    />
+                </fieldset>
+            );
+        } else if (lastArg === 'float') {
+            const valSetterShim = (event: {target: {value: string}}) =>
+                setter(parseFloat(event.target.value));
+            return (
+                <fieldset key={renderKey}>
+                    <label> {label} </label>
+                    <input
+                        type='number'
+                        step='0.01'
+                        defaultValue={val as number}
+                        onChange={valSetterShim}
+                    />
+                </fieldset>
+            );
+        } else if (lastArg === 'string') {
+            const valSetterShim = (event: {target: {value: string}}) => setter(event.target.value);
+            return (
+                <fieldset key={renderKey}>
+                    <label> {label} </label>
+                    <input type='text' defaultValue={val as string} onChange={valSetterShim} />
+                </fieldset>
+            );
+        } else {
+            return (
+                <fieldset key={renderKey}>
+                    <label> {label} </label>
+                    <input value='dont know' />
+                </fieldset>
+            );
+        }
+    } else if (argType === 'colEnum') {
+        const widgetRow = columns.map((colName: string) => {
+            const colSetter = (event: {target: {value: any}}) => {
+                const newColVal = event.target.value;
+                if (_.isString(newColVal)) {
+                    const updatedColDict = replaceAtKey(
+                        val as Record<string, string>,
+                        colName,
+                        newColVal as string
+                    ); // as Record<string, string>
+                    setter(objWithoutNull(updatedColDict, ['null']));
+                }
+            };
+            const colVal = _.get(val, colName, 'null');
+            if (!_.isArray(lastArg)) {
+                return <h3> arg error</h3>;
+            }
+            return (
+                <td key={renderKey + colName}>
+                    <select defaultValue={colVal} onChange={colSetter}>
+                        {lastArg.map((optionVal) => (
+                            <option key={optionVal} value={optionVal}>
+                                {optionVal}
+                            </option>
+                        ))}
+                    </select>
+                </td>
+            );
+        });
+
+        return (
+            <div className='col-enum' key={renderKey}>
+                <table>
+                    <thead>
+                        <tr>
+                            {columns.map((colName) => (
+                                <th key={colName}>{colName}</th>
+                            ))}
+                        </tr>
+                    </thead>
+                    <tbody>
+                        <tr>{widgetRow}</tr>
+                    </tbody>
+                </table>
+            </div>
+        );
+    } else {
+        return <h3> unknown argtype </h3>;
+    }
+};

package/src/buckaroo_js/components/OperationUtils.ts

@@ -0,0 +1,33 @@
+/*
+  used for manipulating the JSON Flavored lisp of operations and commands
+  */
+import _ from 'lodash';
+import {SymbolT, ColEnumArgs, SymbolDf, symDf} from './CommandUtils';
+
+export const sym = (symbolName: string) => {
+    return {symbol: symbolName};
+};
+
+export type Atom = number | string | SymbolT | ColEnumArgs;
+export type SettableArg = number | string | ColEnumArgs;
+
+export type OperationSingleColumn = [SymbolT, SymbolDf, string];
+export type OperationSingleArg = [SymbolT, SymbolDf, string, Atom];
+export type OperationTwoArg = [SymbolT, SymbolDf, string, Atom, Atom];
+export type Operation = OperationSingleColumn | OperationSingleArg | OperationTwoArg;
+
+export type SetOperationFunc = (newCommand: Operation) => void;
+export type SetOperationsFunc = (newCommands: Operation[]) => void;
+
+export type OperationDefaultArgs = Record<string, Operation>;
+
+//const ArgNames = ['Idx', 'label', 'specName', 'extraSpecArgs'];
+export const bakedOperations: Operation[] = [
+    [sym('dropcol'), symDf, 'col1'],
+    [sym('fillna'), symDf, 'col2', 5],
+    [sym('resample'), symDf, 'month', 'monthly', {}]
+];
+
+//this will become OperationEventFunc
+export type OperationEventFunc = (newCommand: Operation) => void;
+export type NoArgEventFunc = () => void;