@datagrok/eda 1.1.25 → 1.1.28

package/src/package.ts

@@ -10,7 +10,7 @@ import {computePCA} from './eda-tools';
 import {addPrefixToEachColumnName, addOneWayAnovaVizualization} from './eda-ui';
 import {testDataForBinaryClassification} from './data-generators';
 import {LINEAR, RBF, POLYNOMIAL, SIGMOID,
-  getTrainedModel, getPrediction, showTrainReport, getPackedModel} from './svm';
+  getTrainedModel, getPrediction, isApplicableSVM, isInteractiveSVM, showTrainReport, getPackedModel} from './svm';
 
 import {PLS_ANALYSIS} from './pls/pls-constants';
 import {runMVA, runDemoMVA, getPlsAnalysis, PlsOutput} from './pls/pls-tools';
@@ -29,6 +29,9 @@ import {DimReductionMethods} from '@datagrok-libraries/ml/src/multi-column-dimen
 import {runKNNImputer} from './missing-values-imputation/ui';
 import {MCLEditor} from '@datagrok-libraries/ml/src/MCL/mcl-editor';
 import {markovCluster} from '@datagrok-libraries/ml/src/MCL/clustering-view';
+import {MCL_OPTIONS_TAG, MCLSerializableOptions} from '@datagrok-libraries/ml/src/MCL';
+
+import {getLinearRegressionParams, getPredictionByLinearRegression, getTestDatasetForLinearRegression} from './regression';
 
 export const _package = new DG.Package();
 
@@ -189,7 +192,7 @@ export function GetMCLEditor(call: DG.FuncCall): void {
           df: params.table, cols: params.columns, metrics: params.distanceMetrics,
           weights: params.weights, aggregationMethod: params.aggreaggregationMethod, preprocessingFuncs: params.preprocessingFunctions,
           preprocessingFuncArgs: params.preprocessingFuncArgs, threshold: params.threshold, maxIterations: params.maxIterations,
-          useWebGPU: params.useWebGPU,
+          useWebGPU: params.useWebGPU, inflate: params.inflateFactor,
         }).call(true);
       }).show();
   } catch (err: any) {
@@ -214,13 +217,46 @@ export function GetMCLEditor(call: DG.FuncCall): void {
 //input: int threshold = 80
 //input: int maxIterations = 10
 //input: bool useWebGPU = false
+//input: double inflate = 2
 //editor: EDA: GetMCLEditor
 export async function MCL(df: DG.DataFrame, cols: DG.Column[], metrics: KnownMetrics[],
   weights: number[], aggregationMethod: DistanceAggregationMethod, preprocessingFuncs: (DG.Func | null | undefined)[],
-  preprocessingFuncArgs: any[], threshold: number = 80, maxIterations: number = 10, useWebGPU: boolean = false,
+  preprocessingFuncArgs: any[], threshold: number = 80, maxIterations: number = 10, useWebGPU: boolean = false, inflate: number = 0,
 ): Promise< DG.ScatterPlotViewer | undefined> {
-  const res = (await markovCluster(df, cols, metrics, weights,
-    aggregationMethod, preprocessingFuncs, preprocessingFuncArgs, threshold, maxIterations, useWebGPU));
+  const tv = grok.shell.tableView(df.name) ?? grok.shell.addTableView(df);
+  const serializedOptions: string = JSON.stringify({
+    cols: cols.map((col) => col.name),
+    metrics: metrics,
+    weights: weights,
+    aggregationMethod: aggregationMethod,
+    preprocessingFuncs: preprocessingFuncs.map((func) => func?.name ?? null),
+    preprocessingFuncArgs: preprocessingFuncArgs,
+    threshold: threshold,
+    maxIterations: maxIterations,
+    useWebGPU: useWebGPU,
+    inflate: inflate,
+  } satisfies MCLSerializableOptions);
+  df.setTag(MCL_OPTIONS_TAG, serializedOptions);
+
+  const sc = tv.addViewer(DG.VIEWER.SCATTER_PLOT, {title: 'MCL', initializationFunction: 'EDA:MCLInitializationFunction'}) as DG.ScatterPlotViewer;
+  return sc;
+}
+
+//name: MCLInitializationFunction
+//input: viewer sc
+export async function MCLInitializationFunction(sc: DG.ScatterPlotViewer) {
+  const df = sc.dataFrame;
+  if (df === null)
+    throw new Error('Data frame of the scatter plot is null');
+  const mclTag = df.getTag(MCL_OPTIONS_TAG);
+  if (!mclTag)
+    throw new Error('MCL options tag on the dataFrame is not found');
+  const options: MCLSerializableOptions = JSON.parse(mclTag);
+  const cols = options.cols.map((colName) => df.columns.byName(colName));
+  const preprocessingFuncs = options.preprocessingFuncs.map((funcName) => funcName ? DG.Func.byName(funcName) : null);
+  const res = await markovCluster(df, cols, options.metrics, options.weights,
+    options.aggregationMethod, preprocessingFuncs, options.preprocessingFuncArgs, options.threshold,
+    options.maxIterations, options.useWebGPU, options.inflate, sc);
   return res?.sc;
 }
 
@@ -298,17 +334,12 @@ export async function testDataLinearNonSeparable(name: string, sigma: number, sa
 //meta.mlname: linear kernel LS-SVM
 //meta.mlrole: train
 //input: dataframe df
-//input: string predict_column
+//input: column predictColumn
 //input: double gamma = 1.0 {category: Hyperparameters}
-//input: bool toShowReport = false {caption: to show report; category: Report}
 //output: dynamic model
-export async function trainLinearKernelSVM(df: DG.DataFrame, predict_column: string,
-  gamma: number, toShowReport: boolean): Promise<any> {
-  const trainedModel = await getTrainedModel({gamma: gamma, kernel: LINEAR}, df, predict_column);
-
-  if (toShowReport)
-    showTrainReport(df, trainedModel);
-
+export async function trainLinearKernelSVM(df: DG.DataFrame, predictColumn: DG.Column,
+  gamma: number): Promise<any> {
+  const trainedModel = await getTrainedModel({gamma: gamma, kernel: LINEAR}, df, predictColumn);
   return getPackedModel(trainedModel);
 }
 
@@ -322,23 +353,52 @@ export async function applyLinearKernelSVM(df: DG.DataFrame, model: any): Promis
   return await getPrediction(df, model);
 }
 
+//name: isApplicableLinearKernelSVM
+//meta.mlname: linear kernel LS-SVM
+//meta.mlrole: isApplicable
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isApplicableLinearKernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isApplicableSVM(df, predictColumn);
+}
+
+//name: isInteractiveLinearKernelSVM
+//meta.mlname: linear kernel LS-SVM
+//meta.mlrole: isInteractive
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isInteractiveLinearKernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isInteractiveSVM(df, predictColumn);
+}
+
+//name: visualizeLinearKernelSVM
+//meta.mlname: linear kernel LS-SVM
+//meta.mlrole: visualize
+//input: dataframe df
+//input: column targetColumn
+//input: column predictColumn
+//input: dynamic model
+//output: dynamic widget
+export async function visualizeLinearKernelSVM(df: DG.DataFrame, targetColumn: DG.Column, predictColumn: DG.Column, model: any): Promise<any> {
+  return showTrainReport(df, model);
+}
+
+
 //name: trainRBFkernelSVM
 //meta.mlname: RBF-kernel LS-SVM
 //meta.mlrole: train
 //input: dataframe df
-//input: string predict_column
+//input: column predictColumn
 //input: double gamma = 1.0 {category: Hyperparameters}
 //input: double sigma = 1.5 {category: Hyperparameters}
-//input: bool toShowReport = false {caption: to show report; category: Report}
 //output: dynamic model
-export async function trainRBFkernelSVM(df: DG.DataFrame, predict_column: string,
-  gamma: number, sigma: number, toShowReport: boolean): Promise<any> {
+export async function trainRBFkernelSVM(df: DG.DataFrame, predictColumn: DG.Column,
+  gamma: number, sigma: number): Promise<any> {
   const trainedModel = await getTrainedModel(
     {gamma: gamma, kernel: RBF, sigma: sigma},
-    df, predict_column);
-
-  if (toShowReport)
-    showTrainReport(df, trainedModel);
+    df, predictColumn);
 
   return getPackedModel(trainedModel);
 }
@@ -353,24 +413,53 @@ export async function applyRBFkernelSVM(df: DG.DataFrame, model: any): Promise<D
   return await getPrediction(df, model);
 }
 
+//name: isApplicableRBFkernelSVM
+//meta.mlname: RBF-kernel LS-SVM
+//meta.mlrole: isApplicable
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isApplicableRBFkernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isApplicableSVM(df, predictColumn);
+}
+
+//name: isInteractiveRBFkernelSVM
+//meta.mlname: RBF-kernel LS-SVM
+//meta.mlrole: isInteractive
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isInteractiveRBFkernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isInteractiveSVM(df, predictColumn);
+}
+
+
+//name: visualizeRBFkernelSVM
+//meta.mlname: RBF-kernel LS-SVM
+//meta.mlrole: visualize
+//input: dataframe df
+//input: column targetColumn
+//input: column predictColumn
+//input: dynamic model
+//output: dynamic widget
+export async function visualizeRBFkernelSVM(df: DG.DataFrame, targetColumn: DG.Column, predictColumn: DG.Column, model: any): Promise<any> {
+  return showTrainReport(df, model);
+}
+
 //name: trainPolynomialKernelSVM
 //meta.mlname: polynomial kernel LS-SVM
 //meta.mlrole: train
 //input: dataframe df
-//input: string predict_column
+//input: column predictColumn
 //input: double gamma = 1.0 {category: Hyperparameters}
 //input: double c = 1 {category: Hyperparameters}
 //input: double d = 2 {category: Hyperparameters}
-//input: bool toShowReport = false {caption: to show report; category: Report}
 //output: dynamic model
-export async function trainPolynomialKernelSVM(df: DG.DataFrame, predict_column: string,
-  gamma: number, c: number, d: number, toShowReport: boolean): Promise<any> {
+export async function trainPolynomialKernelSVM(df: DG.DataFrame, predictColumn: DG.Column,
+  gamma: number, c: number, d: number): Promise<any> {
   const trainedModel = await getTrainedModel(
     {gamma: gamma, kernel: POLYNOMIAL, cParam: c, dParam: d},
-    df, predict_column);
-
-  if (toShowReport)
-    showTrainReport(df, trainedModel);
+    df, predictColumn);
 
   return getPackedModel(trainedModel);
 } // trainPolynomialKernelSVM
@@ -385,24 +474,52 @@ export async function applyPolynomialKernelSVM(df: DG.DataFrame, model: any): Pr
   return await getPrediction(df, model);
 }
 
+//name: isApplicablePolynomialKernelSVM
+//meta.mlname: polynomial kernel LS-SVM
+//meta.mlrole: isApplicable
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isApplicablePolynomialKernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isApplicableSVM(df, predictColumn);
+}
+
+//name: isInteractivePolynomialKernelSVM
+//meta.mlname: polynomial kernel LS-SVM
+//meta.mlrole: isInteractive
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isInteractivePolynomialKernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isInteractiveSVM(df, predictColumn);
+}
+
+//name: visualizePolynomialKernelSVM
+//meta.mlname: polynomial kernel LS-SVM
+//meta.mlrole: visualize
+//input: dataframe df
+//input: column targetColumn
+//input: column predictColumn
+//input: dynamic model
+//output: dynamic widget
+export async function visualizePolynomialKernelSVM(df: DG.DataFrame, targetColumn: DG.Column, predictColumn: DG.Column, model: any): Promise<any> {
+  return showTrainReport(df, model);
+}
+
 //name: trainSigmoidKernelSVM
 //meta.mlname: sigmoid kernel LS-SVM
 //meta.mlrole: train
 //input: dataframe df
-//input: string predict_column
+//input: column predictColumn
 //input: double gamma = 1.0 {category: Hyperparameters}
 //input: double kappa = 1 {category: Hyperparameters}
 //input: double theta = 1 {category: Hyperparameters}
-//input: bool toShowReport = false {caption: to show report; category: Report}
 //output: dynamic model
-export async function trainSigmoidKernelSVM(df: DG.DataFrame, predict_column: string,
-  gamma: number, kappa: number, theta: number, toShowReport: boolean): Promise<any> {
+export async function trainSigmoidKernelSVM(df: DG.DataFrame, predictColumn: DG.Column,
+  gamma: number, kappa: number, theta: number): Promise<any> {
   const trainedModel = await getTrainedModel(
     {gamma: gamma, kernel: SIGMOID, kappa: kappa, theta: theta},
-    df, predict_column);
-
-  if (toShowReport)
-    showTrainReport(df, trainedModel);
+    df, predictColumn);
 
   return getPackedModel(trainedModel);
 } // trainSigmoidKernelSVM
@@ -417,6 +534,38 @@ export async function applySigmoidKernelSVM(df: DG.DataFrame, model: any): Promi
   return await getPrediction(df, model);
 }
 
+//name: isApplicableSigmoidKernelSVM
+//meta.mlname: sigmoid kernel LS-SVM
+//meta.mlrole: isApplicable
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isApplicableSigmoidKernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isApplicableSVM(df, predictColumn);
+}
+
+//name: isInteractiveSigmoidKernelSVM
+//meta.mlname: sigmoid kernel LS-SVM
+//meta.mlrole: isInteractive
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export async function isInteractiveSigmoidKernelSVM(df: DG.DataFrame, predictColumn: DG.Column): Promise<boolean> {
+  return isInteractiveSVM(df, predictColumn);
+}
+
+//name: visualizeSigmoidKernelSVM
+//meta.mlname: sigmoid kernel LS-SVM
+//meta.mlrole: visualize
+//input: dataframe df
+//input: column targetColumn
+//input: column predictColumn
+//input: dynamic model
+//output: dynamic widget
+export async function visualizeSigmoidKernelSVM(df: DG.DataFrame, targetColumn: DG.Column, predictColumn: DG.Column, model: any): Promise<any> {
+  return showTrainReport(df, model);
+}
+
 //top-menu: ML | Analyze | ANOVA...
 //name: ANOVA
 //description: One-way analysis of variances (ANOVA) determines whether the examined factor has a significant impact on the studied feature.
@@ -436,3 +585,104 @@ export function anova(table: DG.DataFrame, factor: DG.Column, feature: DG.Column
 export function kNNImputation() {
   runKNNImputer();
 }
+
+//name: KNN imputation for a table
+//desription: Missing values imputation using the k-nearest neighbors method for a given table
+//input: dataframe table
+export async function kNNImputationForTable(table: DG.DataFrame) {
+  await runKNNImputer(table);
+}
+
+//name: linearRegression
+//description: Linear Regression demo
+//input: dataframe table
+//input: column_list features {type: numerical}
+//input: column target {type: numerical}
+//input: bool plot = true {caption: plot}
+export async function linearRegression(table: DG.DataFrame, features: DG.ColumnList, target: DG.Column, plot: boolean): Promise<void> {
+  const t1 = performance.now();
+  const params = await getLinearRegressionParams(features, target);
+  const t2 = performance.now();
+  console.log(`Fit: ${t2 - t1} ms.`);
+  const prediction = getPredictionByLinearRegression(features, params);
+  console.log(`Predict: ${performance.now() - t2} ms.`);
+
+  prediction.name = table.columns.getUnusedName(prediction.name);
+
+  table.columns.add(prediction);
+
+  if (plot) {
+    const view = grok.shell.tableView(table.name);
+    view.addViewer(DG.VIEWER.SCATTER_PLOT, {
+      xColumnName: target.name,
+      yColumnName: prediction.name,
+      showRegressionLine: true,
+    });
+  }
+}
+
+//name: generateDatasetForLinearRegressionTest
+//description: Create demo dataset for linear regression
+//input: int rowCount = 10000 {min: 1000; max: 10000000; step: 10000}
+//input: int colCount = 10 {min: 1; max: 1000; step: 10}
+//input: double featuresScale = 10 {min: -1000; max: 1000; step: 10}
+//input: double featuresBias = 10 {min: -1000; max: 1000; step: 10}
+//input: double paramsScale = 10 {min: -1000; max: 1000; step: 10}
+//input: double paramsBias = 10 {min: -1000; max: 1000; step: 10}
+//output: dataframe table
+export function generateDatasetForLinearRegressionTest(rowCount: number, colCount: number,
+  featuresScale: number, featuresBias: number, paramsScale: number, paramsBias: number): DG.DataFrame {
+  return getTestDatasetForLinearRegression(rowCount, colCount, featuresScale, featuresBias, paramsScale, paramsBias);
+}
+
+//name: trainLinearRegression
+//meta.mlname: Linear Regression
+//meta.mlrole: train
+//input: dataframe df
+//input: column predictColumn
+//output: dynamic model
+export async function trainLinearRegression(df: DG.DataFrame, predictColumn: DG.Column): Promise<Uint8Array> {
+  const features = df.columns;
+  const params = await getLinearRegressionParams(features, predictColumn);
+
+  return new Uint8Array(params.buffer);
+}
+
+//name: applyLinearRegression
+//meta.mlname: Linear Regression
+//meta.mlrole: apply
+//input: dataframe df
+//input: dynamic model
+//output: dataframe table
+export function applyLinearRegression(df: DG.DataFrame, model: any): DG.DataFrame {
+  const features = df.columns;
+  const params = new Float32Array((model as Uint8Array).buffer);
+  return DG.DataFrame.fromColumns([getPredictionByLinearRegression(features, params)]);
+}
+
+//name: isApplicableLinearRegression
+//meta.mlname: Linear Regression
+//meta.mlrole: isApplicable
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export function isApplicableLinearRegression(df: DG.DataFrame, predictColumn: DG.Column): boolean {
+  for (const col of df.columns) {
+    if (!col.matches('numerical'))
+      return false;
+  }
+  if (!predictColumn.matches('numerical'))
+    return false;
+
+  return true;
+}
+
+//name: isInteractiveLinearRegression
+//meta.mlname: Linear Regression
+//meta.mlrole: isInteractive
+//input: dataframe df
+//input: column predictColumn
+//output: bool result
+export function isInteractiveLinearRegression(df: DG.DataFrame, predictColumn: DG.Column): boolean {
+  return df.rowCount <= 100000;
+}

package/src/pls/pls-tools.ts

@@ -6,7 +6,7 @@ import * as DG from 'datagrok-api/dg';
 
 import {PLS_ANALYSIS, ERROR_MSG, TITLE, HINT, LINK, COMPONENTS, INT, TIMEOUT,
   RESULT_NAMES, WASM_OUTPUT_IDX, RADIUS, LINE_WIDTH, COLOR, X_COORD, Y_COORD,
-  DEMO_INTRO_MD, DEMO_RESULTS_MD, DELAY, DEMO_RESULTS} from './pls-constants';
+  DEMO_INTRO_MD, DEMO_RESULTS_MD, DEMO_RESULTS} from './pls-constants';
 import {checkWasmDimensionReducerInputs, checkColumnType, checkMissingVals} from '../utils';
 import {_partialLeastSquareRegressionInWebWorker} from '../../wasm/EDAAPI';
 import {carsDataframe} from '../data-generators';
@@ -141,7 +141,7 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
   const scoresScatter = DG.Viewer.scatterPlot(input.table, {
     title: TITLE.SCORES,
     xColumnName: plsCols[0].name,
-    yColumnName: (plsCols.length > 1) ? plsCols[1].name : result.uScores[0],
+    yColumnName: (plsCols.length > 1) ? plsCols[1].name : result.uScores[0].name,
     markerType: DG.MARKER_TYPE.CIRCLE,
     labels: input.names?.name,
     help: LINK.SCORES,
@@ -161,7 +161,8 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
       min: -radius,
       max: radius,
       color: COLOR.CIRCLE,
-  })};
+    });
+  };
 
   scoreNames.forEach((xName) => {
     const x = '${' + xName + '}';
@@ -226,12 +227,13 @@ async function performMVA(input: PlsInput, analysisType: PLS_ANALYSIS): Promise<
 
   // emphasize viewers in the demo case
   if (analysisType === PLS_ANALYSIS.DEMO) {
-    const pages = [predictVsReferScatter, scoresScatter, loadingsScatter, regrCoeffsBar, explVarsBar].map((viewer, idx) => {
-      return {
-        text: DEMO_RESULTS[idx].text,
-        showNextTo: viewer.root,
-      }
-    });
+    const pages = [predictVsReferScatter, scoresScatter, loadingsScatter, regrCoeffsBar, explVarsBar]
+      .map((viewer, idx) => {
+        return {
+          text: DEMO_RESULTS[idx].text,
+          showNextTo: viewer.root,
+        };
+      });
 
     const wizard = ui.hints.addTextHint({title: TITLE.EXPLORE, pages: pages});
     wizard.helpUrl = LINK.MVA;
@@ -281,17 +283,16 @@ export async function runMVA(analysisType: PLS_ANALYSIS): Promise<void> {
 
   // responce (to predict)
   let predict = numCols[numCols.length - 1];
-  const predictInput = ui.columnInput(TITLE.PREDICT, table, predict, () => {
+  const predictInput = ui.input.column(TITLE.PREDICT, {table: table, value: predict, onValueChanged: () => {
     predict = predictInput.value!;
     updateIputs();
-  },
-  {filter: (col: DG.Column) => isValidNumeric(col)},
+  }, filter: (col: DG.Column) => isValidNumeric(col)},
   );
   predictInput.setTooltip(HINT.PREDICT);
 
   // predictors (features)
   let features: DG.Column[];
-  const featuresInput = ui.columnsInput(TITLE.USING, table, () => {}, {available: numColNames});
+  const featuresInput = ui.input.columns(TITLE.USING, {table: table, available: numColNames});
   featuresInput.onInput(() => updateIputs());
   featuresInput.setTooltip(HINT.FEATURES);
 
@@ -334,8 +335,8 @@ export async function runMVA(analysisType: PLS_ANALYSIS): Promise<void> {
 
   // names of samples
   let names = (strCols.length > 0) ? strCols[0] : null;
-  const namesInputs = ui.columnInput(TITLE.NAMES, table, names, () => names = predictInput.value,
-    {filter: (col: DG.Column) => col.type === DG.COLUMN_TYPE.STRING},
+  const namesInputs = ui.input.column(TITLE.NAMES, {table: table, value: names!, onValueChanged: () => names = predictInput.value,
+    filter: (col: DG.Column) => col.type === DG.COLUMN_TYPE.STRING},
   );
   namesInputs.setTooltip(HINT.NAMES);
   namesInputs.root.hidden = (strCols.length === 0) || (analysisType === PLS_ANALYSIS.COMPUTE_COMPONENTS);