@datagrok/eda 1.4.1 → 1.4.2

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@datagrok/eda",
   "friendlyName": "EDA",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "Exploratory Data Analysis Tools",
   "dependencies": {
     "@datagrok-libraries/math": "^1.2.6",
     "@datagrok-libraries/ml": "^6.8.3",
     "@datagrok-libraries/tutorials": "^1.6.1",
-    "@datagrok-libraries/utils": "^4.5.0",
+    "@datagrok-libraries/utils": "^4.5.7",
     "@keckelt/tsne": "^1.0.2",
     "@webgpu/types": "^0.1.40",
     "cash-dom": "^8.1.1",

package/src/package.ts

@@ -278,6 +278,7 @@ export async function PLS(table: DG.DataFrame, features: DG.ColumnList, predict:
     features: features,
     predict: predict,
     components: components,
+    isQuadratic: false,
     names: names,
   });
 }

package/src/pls/pls-constants.ts

@@ -13,6 +13,12 @@ export enum ERROR_MSG {
   NO_COLS = 'No numeric columns without missing values',
   ONE_COL = 'No columns to be used as features (just one numeric columns without missing values)',
   EMPTY_DF = 'Dataframe is empty',
+  PREDICT = 'Predictors must not contain a response variable',
+  ENOUGH = 'Not enough of features',
+  COMP_LIN_PLS = 'Components count must be less than the number of features',
+  COMP_QUA_PLS = 'Too large components count for the quadratic PLS regression',
+  COMPONENTS = 'Components count must be greater than 1',
+  INV_INP = 'Invalid inputs',
 }
 
 /** Widget titles */
@@ -37,6 +43,7 @@ export enum TITLE {
   FEATURES = 'Feature names',
   BROWSE = 'Browse',
   ANALYSIS = 'Features Analysis',
+  QUADRATIC = 'Quadratic',
 }
 
 /** Tooltips */
@@ -47,6 +54,7 @@ export enum HINT {
   PLS = 'Compute PLS components',
   MVA = 'Perform multivariate analysis',
   NAMES = 'Names of data samples',
+  QUADRATIC = 'Specifies whether to include squared terms as additional predictors in the PLS model',
 }
 
 /** Links to help */
@@ -96,6 +104,9 @@ export const DELAY = 2000;
 export enum COLOR {
   AXIS = '#838383',
   CIRCLE = '#0000FF',
+  INVALID = '#EB6767',
+  VALID_TEXT = '#4d5261',
+  VALID_LINE = '#dbdcdf',
 };
 
 /** Intro markdown for demo app */
@@ -131,7 +142,7 @@ export const DEMO_RESULTS = [
   },
   {
     caption: TITLE.REGR_COEFS,
-    text: 'Parameters of the obtained linear model: features make different contribution to the prediction.',
+    text: 'Parameters of the obtained model: features make different contribution to the prediction.',
   },
   {
     caption: TITLE.EXPL_VAR,

package/src/pls/pls-ml.ts

@@ -121,6 +121,7 @@ export class PlsModel {
       predict: target,
       components: components,
       names: undefined,
+      isQuadratic: false,
     });
 
     // 1. Names of features

package/src/pls/pls-tools.ts

@@ -30,9 +30,12 @@ export type PlsInput = {
   features: DG.ColumnList,
   predict: DG.Column,
   components: number,
+  isQuadratic: boolean,
   names : DG.Column | undefined,
 };
 
+type TypedArray = Int32Array | Float32Array | Uint32Array | Float64Array;
+
 /** Return lines */
 export function getLines(names: string[]): DG.FormulaLine[] {
   const lines: DG.FormulaLine[] = [];
@@ -113,12 +116,63 @@ function debiasedPrediction(features: DG.ColumnList, params: DG.Column,
   return DG.Column.fromFloat32Array('Debiased', debiased, samples);
 }
 
+/** Return an input for the quadratic PLS regression */
+function getQuadraticPlsInput(input: PlsInput): PlsInput {
+  if (!input.isQuadratic)
+    return input;
+
+  const cols: DG.Column[] = input.features.toList();
+  const colsCount = cols.length;
+  const rowsCount = input.table.rowCount;
+  const quadrCols: DG.Column[] = [];
+  let col1: DG.Column;
+  let raw1: TypedArray;
+  let col2: DG.Column;
+  let raw2: TypedArray;
+  let qaudrRaw: Float32Array;
+
+  for (let i = 0; i < colsCount; ++i) {
+    col1 = cols[i];
+    raw1 = col1.getRawData();
+
+    for (let j = i; j < colsCount; ++j) {
+      col2 = cols[j];
+      raw2 = col2.getRawData();      
+      qaudrRaw = new Float32Array(rowsCount);
+
+      for (let k = 0; k < rowsCount; ++k)
+        qaudrRaw[k] = raw1[k] * raw2[k];
+
+      const quadrCol = DG.Column.fromFloat32Array(`${col1.name} x ${col2.name}`, qaudrRaw);
+
+      if (quadrCol.stats.stdev > 0)
+        quadrCols.push(quadrCol);
+    }
+  }
+
+  const extendedTable = DG.DataFrame.fromColumns(cols.concat(quadrCols));
+
+  return {
+    table: extendedTable,
+    features: extendedTable.columns,
+    isQuadratic: true,
+    names: input.names,
+    predict: input.predict,
+    components: input.components,
+  };
+}
+
 /** Perform multivariate analysis using the PLS regression */
 async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<void> {
+  const sourceTable = input.table;
+
+  if (input.isQuadratic)
+    input = getQuadraticPlsInput(input);
+
   const result = await getPlsAnalysis(input);
 
   const plsCols = result.tScores;
-  const cols = input.table.columns;
+  const cols = sourceTable.columns;
   const features = input.features;
   const featuresNames = features.names();
   const prefix = (analysisType === PLS_ANALYSIS.COMPUTE_COMPONENTS) ? RESULT_NAMES.PREFIX : TITLE.XSCORE;
@@ -132,7 +186,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
   if (analysisType === PLS_ANALYSIS.COMPUTE_COMPONENTS)
     return;
 
-  const view = grok.shell.tableView(input.table.name);
+  const view = grok.shell.tableView(sourceTable.name);
 
   // 0.1 Buffer table
   const loadingsRegrCoefsTable = DG.DataFrame.fromColumns([
@@ -140,7 +194,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
     result.regressionCoefficients,
   ]);
 
-  loadingsRegrCoefsTable.name = `${input.table.name}(${TITLE.ANALYSIS})`;
+  loadingsRegrCoefsTable.name = `${sourceTable.name}(${TITLE.ANALYSIS})`;
   grok.shell.addTable(loadingsRegrCoefsTable);
 
   // 0.2. Add X-Loadings
@@ -154,7 +208,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
   const pred = debiasedPrediction(features, result.regressionCoefficients, input.predict, result.prediction);
   pred.name = cols.getUnusedName(`${input.predict.name} ${RESULT_NAMES.SUFFIX}`);
   cols.add(pred);
-  const predictVsReferScatter = view.addViewer(DG.Viewer.scatterPlot(input.table, {
+  const predictVsReferScatter = view.addViewer(DG.Viewer.scatterPlot(sourceTable, {
     title: TITLE.MODEL,
     xColumnName: input.predict.name,
     yColumnName: pred.name,
@@ -203,7 +257,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
   });
 
   // 4.2) create scatter
-  const scoresScatter = DG.Viewer.scatterPlot(input.table, {
+  const scoresScatter = DG.Viewer.scatterPlot(sourceTable, {
     title: TITLE.SCORES,
     xColumnName: plsCols[0].name,
     yColumnName: (plsCols.length > 1) ? plsCols[1].name : result.uScores[0].name,
@@ -224,7 +278,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
   //      here, we use notations from this paper
   const q = result.yLoadings.getRawData();
   const p = result.xLoadings.map((col) => col.getRawData());
-  const n = input.table.rowCount;
+  const n = sourceTable.rowCount;
   const m = featuresNames.length;
   const A = input.components;
   const yExplVars = new Float32Array(A);
@@ -249,7 +303,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
     DG.Column.fromFloat32Array(input.predict.name, yExplVars),
   ]);
 
-  explVarsDF.name = `${input.table.name}(${TITLE.EXPL_VAR})`;
+  explVarsDF.name = `${sourceTable.name}(${TITLE.EXPL_VAR})`;
   grok.shell.addTable(explVarsDF);
 
   xExplVars.forEach((arr, idx) => explVarsDF.columns.add(DG.Column.fromFloat32Array(featuresNames[idx], arr)));
@@ -318,33 +372,65 @@ export async function runMVA(analysisType: PLS_ANALYSIS): Promise<void> {
     return;
   }
 
-  // responce (to predict)
+  let features: DG.Column[] = numCols.slice(0, numCols.length - 1);
   let predict = numCols[numCols.length - 1];
-  const predictInput = ui.input.column(TITLE.PREDICT, {table: table, value: predict, onValueChanged: (value) => {
-    predict = value;
-    updateIputs();
-  }, filter: (col: DG.Column) => isValidNumeric(col)},
-  );
-  predictInput.setTooltip(HINT.PREDICT);
+  let components = min(numColNames.length - 1, COMPONENTS.DEFAULT as number);
+  let isQuadratic = false;
+
+  const isPredictValid = () => {
+    for (const col of features)
+      if (col.name === predict.name)
+        return false;
+    return true;
+  };
+
+  const isCompConsistent = () => {
+    if (components < 1)
+      return false;
+
+    const n = features.length;
+
+    if (isQuadratic)      
+      return components <= (n + 1) * n / 2 + n;
+
+    return components <= n;
+  }
+
+  // response (to predict)
+  const predictInput = ui.input.column(TITLE.PREDICT, {
+    table: table,
+    value: predict,
+    nullable: false,
+    onValueChanged: (value) => {
+      predict = value;
+      updateIputs();
+    },
+    filter: (col: DG.Column) => isValidNumeric(col),
+    tooltipText: HINT.PREDICT,
+  });
 
   // predictors (features)
-  let features: DG.Column[];
-  const featuresInput = ui.input.columns(TITLE.USING, {table: table, available: numColNames});
-  featuresInput.onInput.subscribe(() => updateIputs());
-  featuresInput.setTooltip(HINT.FEATURES);
+  const featuresInput = ui.input.columns(TITLE.USING, {
+    table: table,
+    available: numColNames,
+    value: features,
+    onValueChanged: (val) => {
+      features = val;
+      updateIputs();      
+    },
+    tooltipText: HINT.FEATURES,
+  });
 
   // components count
-  let components = min(numColNames.length - 1, COMPONENTS.DEFAULT as number);
-  const componentsInput = ui.input.forProperty(DG.Property.fromOptions({
-    name: TITLE.COMPONENTS,
-    inputType: INT,
-    defaultValue: components,
-    //@ts-ignore
+  const componentsInput = ui.input.int(TITLE.COMPONENTS, {
+    value: components,
     showPlusMinus: true,
-    min: COMPONENTS.MIN,
-  }));
-  componentsInput.onInput.subscribe(() => updateIputs());
-  componentsInput.setTooltip(HINT.COMPONENTS);
+    onValueChanged: (val) => {
+      components = val;
+      updateIputs();
+    },
+    tooltipText: HINT.COMPONENTS,
+  });
 
   let dlgTitle: string;
   let dlgHelpUrl: string;
@@ -360,14 +446,57 @@ export async function runMVA(analysisType: PLS_ANALYSIS): Promise<void> {
     dlgRunBtnTooltip = HINT.MVA;
   }
 
-  const updateIputs = () => {
-    featuresInput.value = featuresInput.value.filter((col) => col !== predict);
-    features = featuresInput.value;
-
-    componentsInput.value = min(max(componentsInput.value ?? components, COMPONENTS.MIN), features.length);
-    components = componentsInput.value;
+  const setStyle = (valid: boolean, element: HTMLElement, tooltip: string, errorMsg: string) => {
+    if (valid) {
+      element.style.color = COLOR.VALID_TEXT;
+      element.style.borderBottomColor = COLOR.VALID_LINE;
+      ui.tooltip.bind(element, tooltip);
+    } else {
+      element.style.color = COLOR.INVALID;
+      element.style.borderBottomColor = COLOR.INVALID;
+      ui.tooltip.bind(element, () => {
+        const hint = ui.label(tooltip);
+        const err = ui.label(errorMsg);
+        err.style.color = COLOR.INVALID;
+        return ui.divV([hint, err]);
+      });
+    }    
+  };
 
-    dlg.getButton(TITLE.RUN).disabled = (features.length === 0) || (components <= 0);
+  const updateIputs = () => {
+    const predValid = isPredictValid();
+    let compValid: boolean;
+
+    if (predValid) {
+      setStyle(true, predictInput.input, HINT.PREDICT, '');
+      setStyle(true, featuresInput.input, HINT.FEATURES, '');
+    } else {
+      setStyle(false, predictInput.input, HINT.PREDICT, ERROR_MSG.PREDICT);
+      setStyle(false, featuresInput.input, HINT.FEATURES, ERROR_MSG.PREDICT);
+    }
+
+    if (components < 1) {
+      setStyle(false, componentsInput.input, HINT.COMPONENTS, ERROR_MSG.COMPONENTS);
+      compValid = false;  
+    } else {
+      compValid = isCompConsistent();
+
+      if (compValid) {
+        setStyle(true, componentsInput.input, HINT.COMPONENTS, '');
+        if (predValid)
+          setStyle(true, featuresInput.input, HINT.FEATURES, '');
+      } else {
+        const errMsg = isQuadratic ? ERROR_MSG.COMP_QUA_PLS : ERROR_MSG.COMP_LIN_PLS;
+        setStyle(false, componentsInput.input, HINT.COMPONENTS, errMsg);
+        setStyle(false, featuresInput.input, HINT.FEATURES, ERROR_MSG.ENOUGH);
+      }
+    }
+
+    const isValid = predValid && compValid;
+
+    dlg.getButton(TITLE.RUN).disabled = !isValid;
+
+    return isValid;
   };
 
   // names of samples
@@ -381,8 +510,18 @@ export async function runMVA(analysisType: PLS_ANALYSIS): Promise<void> {
   namesInputs.setTooltip(HINT.NAMES);
   namesInputs.root.hidden = (strCols.length === 0) || (analysisType === PLS_ANALYSIS.COMPUTE_COMPONENTS);
 
+  // quadratic/linear model
+  const isQuadraticInput = ui.input.bool(TITLE.QUADRATIC, {
+    value: isQuadratic,
+    tooltipText: HINT.QUADRATIC,
+    onValueChanged: (val) => {
+      isQuadratic = val;
+      updateIputs();
+    },
+  });
+
   const dlg = ui.dialog({title: dlgTitle, helpUrl: dlgHelpUrl})
-    .add(ui.form([predictInput, featuresInput, componentsInput, namesInputs]))
+    .add(ui.form([predictInput, featuresInput, componentsInput, isQuadraticInput, namesInputs]))
     .addButton(TITLE.RUN, async () => {
       dlg.close();
 
@@ -391,6 +530,7 @@ export async function runMVA(analysisType: PLS_ANALYSIS): Promise<void> {
         features: DG.DataFrame.fromColumns(features).columns,
         predict: predict,
         components: components,
+        isQuadratic: isQuadratic,
         names: names,
       }, analysisType);
     }, undefined, dlgRunBtnTooltip)

package/src/regression.ts

@@ -179,6 +179,7 @@ async function getLinearRegressionParamsUsingPLS(features: DG.ColumnList,
     predict: targets,
     components: components,
     names: undefined,
+    isQuadratic: false,
   });
 
   return plsAnalysis.regressionCoefficients.getRawData() as Float32Array;

package/src/tests/linear-methods-tests.ts

@@ -61,6 +61,7 @@ category('Partial least squares regression', () => {
       predict: cols.byIndex(COLS - 1),
       components: COMPONENTS,
       names: undefined,
+      isQuadratic: false,
     });
   }, {timeout: TIMEOUT, benchmark: true});
 
@@ -76,6 +77,7 @@ category('Partial least squares regression', () => {
       features: cols,
       predict: target,
       components: COMPONENTS,
+      isQuadratic: false,
       names: undefined,
     });