@datagrok/eda 1.0.3 → 1.1.1

package/package.json

@@ -1,49 +1,50 @@
 {
-    "name": "@datagrok/eda",
-    "friendlyName": "EDA",
-    "version": "1.0.3",
-    "description": "Eploratory Data Analysis tools",
-    "dependencies": {
-        "datagrok-api": "latest",
-        "cash-dom": "latest",
-        "dayjs": "latest",
-        "@datagrok-libraries/utils": "latest",
-        "@datagrok-libraries/tutorials": "^1.2.1"
-    },
-    "author": {
-        "name": "Viktor Makarichev",
-        "email": "vmakarichev@datagrok.ai"
-    },
-    "devDependencies": {
-        "webpack": "latest",
-        "webpack-cli": "latest",
-        "ts-loader": "latest",
-        "typescript": "latest"
-    },
-    "scripts": {
-        "link-all": "",
-        "debug-eda": "webpack && grok publish",
-        "release-eda": "webpack && grok publish --release",
-        "build-eda": "webpack",
-        "build": "webpack",
-        "debug-eda-dev": "webpack && grok publish dev",
-        "release-eda-dev": "webpack && grok publish dev --release",
-        "debug-eda-local": "webpack && grok publish local",
-        "release-eda-local": "webpack && grok publish local --release"
-    },
-    "canEdit": [
-        "Developers"
-    ],
-    "canView": [
-        "All users"
-    ],
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/datagrok-ai/public.git",
-        "directory": "packages/EDA"
-    },
-    "category": "Machine Learning",
-    "sources": [
-        "wasm/EDA.js"
-    ]
+  "name": "@datagrok/eda",
+  "friendlyName": "EDA",
+  "version": "1.1.1",
+  "description": "Exploratory Data Analysis Tools",
+  "dependencies": {
+    "datagrok-api": "latest",
+    "cash-dom": "latest",
+    "dayjs": "latest",
+    "@datagrok-libraries/utils": "latest",
+    "@datagrok-libraries/tutorials": "^1.3.6"
+  },
+  "author": {
+    "name": "Viktor Makarichev",
+    "email": "vmakarichev@datagrok.ai"
+  },
+  "devDependencies": {
+    "webpack": "latest",
+    "webpack-cli": "latest",
+    "ts-loader": "latest",
+    "typescript": "latest"
+  },
+  "scripts": {
+    "link-all": "npm link datagrok-api @datagrok-libraries/utils @datagrok-libraries/tutorials",
+    "debug-eda": "webpack && grok publish",
+    "release-eda": "webpack && grok publish --release",
+    "build-eda": "webpack",
+    "build": "webpack",
+    "debug-eda-dev": "webpack && grok publish dev",
+    "release-eda-dev": "webpack && grok publish dev --release",
+    "debug-eda-local": "webpack && grok publish local",
+    "release-eda-local": "webpack && grok publish local --release",
+    "build-all": "npm --prefix ./../../js-api run build && npm --prefix ./../../libraries/utils run build && npm --prefix ./../../libraries/tutorials run build && npm run build"
+  },
+  "canEdit": [
+    "Developers"
+  ],
+  "canView": [
+    "All users"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/datagrok-ai/public.git",
+    "directory": "packages/EDA"
+  },
+  "category": "Machine Learning",
+  "sources": [
+    "wasm/EDA.js"
+  ]
 }

package/src/EDAui.ts

@@ -7,16 +7,16 @@ import * as DG from 'datagrok-api/dg';
 // Rename PCA columns
 export function renamePCAcolumns(pcaTable: DG.DataFrame): DG.DataFrame {  
   for (const col of pcaTable.columns.toList()) 
-    col.name = '_PCA' + col.name;
+    col.name = 'PCA' + col.name;
 
   return pcaTable;
 }
 
 // Predicted vs Reference scatter plot
-export function predictedVersusReferenceScatterPlot(reference: DG.Column, prediction: DG.Column): DG.Viewer {  
+export function predictedVersusReferenceScatterPlot(samplesNames: DG.Column, reference: DG.Column, prediction: DG.Column): DG.Viewer {  
   prediction.name = reference.name + '(predicted)';
  
-  let dfReferencePrediction = DG.DataFrame.fromColumns([reference, prediction]);
+  let dfReferencePrediction = DG.DataFrame.fromColumns([samplesNames, reference, prediction]);
   dfReferencePrediction.name = 'Reference vs. Predicted';
    
   return DG.Viewer.scatterPlot(dfReferencePrediction, 
@@ -24,7 +24,8 @@ export function predictedVersusReferenceScatterPlot(reference: DG.Column, predic
       x: reference.name,
       y: prediction.name,
       showRegressionLine: true,
-      markerType: 'circle'
+      markerType: 'circle',
+      labels: samplesNames.name
      });  
 }
 
@@ -46,9 +47,9 @@ export function regressionCoefficientsBarChart(features: DG.ColumnList, regressi
 }
 
 // Scores Scatter Plot
-export function scoresScatterPlot(xScores: Array<DG.Column>, yScores: Array<DG.Column>): DG.Viewer {
+export function scoresScatterPlot(samplesNames: DG.Column, xScores: Array<DG.Column>, yScores: Array<DG.Column>): DG.Viewer {
   
-  let scoresColumns = [];
+  let scoresColumns = [samplesNames];
 
   for (let i = 0; i < xScores.length; i++) {
     xScores[i].name = `x.score.t${i+1}`;
@@ -63,12 +64,15 @@ export function scoresScatterPlot(xScores: Array<DG.Column>, yScores: Array<DG.C
   let scores = DG.DataFrame.fromColumns(scoresColumns);
   scores.name = 'Scores';
   //grok.shell.addTableView(scores);
+
+  const index = xScores.length > 1 ? 1 : 0;
  
   return DG.Viewer.scatterPlot(scores, 
     { title: scores.name,
       x: xScores[0].name,
-      y: yScores[0].name,      
-      markerType: 'circle'
+      y: xScores[index].name,      
+      markerType: 'circle',
+      labels: samplesNames.name
      });
 }
 
@@ -100,19 +104,19 @@ export function loadingScatterPlot(features: DG.ColumnList, xLoadings: Array<DG.
 }
 
 // Add PLS visualization
-export function addPLSvisualization(table: DG.DataFrame, features: DG.ColumnList, predict: DG.Column, plsOutput: any): void {
+export function addPLSvisualization(table: DG.DataFrame, samplesNames: DG.Column, features: DG.ColumnList, predict: DG.Column, plsOutput: any): void {
 
   let view = grok.shell.getTableView(table.name);
  
   // 1. Predicted vs Reference scatter plot
-  view.addViewer(predictedVersusReferenceScatterPlot(predict, plsOutput[0]));
+  view.addViewer(predictedVersusReferenceScatterPlot(samplesNames, predict, plsOutput[0]));
   
   // 2. Regression Coefficients Bar Chart
-  view.addViewer(regressionCoefficientsBarChart(features, plsOutput[1])); 
+  view.addViewer(regressionCoefficientsBarChart(features, plsOutput[1]));   
  
-  // 3. Scores Scatter Plot
-  view.addViewer(scoresScatterPlot(plsOutput[2], plsOutput[3]));
- 
-  // 4. Loading Scatter Plot
+  // 3. Loading Scatter Plot
   view.addViewer(loadingScatterPlot(features, plsOutput[4]));  
+
+  // 4. Scores Scatter Plot
+  view.addViewer(scoresScatterPlot(samplesNames, plsOutput[2], plsOutput[3]));
 }

package/src/package-test.ts

@@ -1,12 +1,15 @@
 import * as DG from "datagrok-api/dg";
-import {runTests, tests} from '@datagrok-libraries/utils/src/test';
+import {runTests, tests, TestContext} from '@datagrok-libraries/utils/src/test';
 
 export let _package = new DG.Package();
 export {tests};
 
 //name: test
+//input: string category {optional: true}
+//input: string test {optional: true}
+//input: object testContext {optional: true}
 //output: dataframe result
-export async function test(): Promise<DG.DataFrame> {
-  let data = await runTests();
+export async function test(category: string, test: string, testContext: TestContext): Promise<DG.DataFrame> {
+  const data = await runTests({category, test, testContext});
   return DG.DataFrame.fromObjects(data)!;
 }

package/src/package.ts

@@ -7,8 +7,8 @@ import {DemoScript} from '@datagrok-libraries/tutorials/src/demo-script';
 
 import {_initEDAAPI} from '../wasm/EDAAPI';
 import {computePCA, computePLS} from './EDAtools';
-import {renamePCAcolumns, addPLSvisualization} from './EDAui';
-import {demoPLS} from './demos';
+import {renamePCAcolumns, addPLSvisualization, regressionCoefficientsBarChart, 
+  scoresScatterPlot, predictedVersusReferenceScatterPlot} from './EDAui';
 import {carsDataframe, testDataForBinaryClassification} from './dataGenerators';
 import {LINEAR, RBF, POLYNOMIAL, SIGMOID, 
   getTrainedModel, getPrediction, showTrainReport, getPackedModel} from './svm';
@@ -25,7 +25,7 @@ export async function init(): Promise<void> {
   await _initEDAAPI();
 }
 
-//top-menu: Tools | Data Science | PCA
+//top-menu: Tools | Data Science | Principal Component Analysis...
 //name: PCA
 //description: Principal component analysis (PCA).
 //input: dataframe table
@@ -40,50 +40,70 @@ export async function PCA(table: DG.DataFrame, features: DG.ColumnList, componen
   return renamePCAcolumns(await computePCA(table, features, components, center, scale));
 }
 
-//top-menu: Tools | Data Science | PLS
-//name: PLS
-//description: Partial least square regression (PLS).
+//top-menu: Tools | Data Science | Multivariate Analysis (PLS)...
+//name: Multivariate Analysis (PLS)
+//description: Multidimensional data analysis using partial least squares (PLS) regression. It reduces the predictors to a smaller set of uncorrelated components and performs least squares regression on them.
 //input: dataframe table
+//input: column names
 //input: column_list features
 //input: column predict
 //input: int components = 3
-export async function PLS(table: DG.DataFrame, features: DG.ColumnList, predict: DG.Column, components: number): Promise<void> {
+export async function PLS(table: DG.DataFrame, names: DG.Column, features: DG.ColumnList, 
+  predict: DG.Column, components: number): Promise<void> 
+{
   const plsResults = await computePLS(table, features, predict, components);
-  addPLSvisualization(table, features, predict, plsResults);
+  addPLSvisualization(table, names, features, predict, plsResults);
 }
 
 //name: MVA demo
-//description: Multivariate analysis (PLS) demo.
-//meta.demoPath: Data analysis | Multivariate analysis
-export async function demoScript(): Promise<any>  {
-  const demoScript = new DemoScript('Multivariate analysis', 
-    'Provides partial least sqaure regression analysis of the given data.'); 
+//description: Multidimensional data analysis using partial least squares (PLS) regression. It reduces the predictors to a smaller set of uncorrelated components and performs least squares regression on them.
+//meta.demoPath: Compute | Multivariate analysis
+//meta.isDemoScript: True
+export async function demoMultivariateAnalysis(): Promise<any>  {
+  const demoScript = new DemoScript('Partial least squares regression', 
+    'Analysis of multidimensional data.'); 
   
   const cars = carsDataframe();
 
   const components = 3;
+  const names = cars.columns.byName('model');
   const predict = cars.columns.byName('price');
   const features = cars.columns.remove('price').remove('model');
   const plsOutput = await computePLS(cars, features, predict, components);  
 
   const sourceCars = carsDataframe();
-  grok.shell.addTableView(sourceCars);
   sourceCars.name = 'Cars';
-  let view = grok.shell.getTableView(sourceCars.name);
+  let view: any;
+  let dialog: any;
 
   await demoScript
-    .step('Run', async () => {}, {description: 'Test dataframe is loaded, and multivariate analysis is performed.', delay: 0})
-    .step('Study', async () => {addPLSvisualization(sourceCars, features, predict, plsOutput)}, {description: 'Investigate results.', delay: 4000})  
-    .step('Try', async () => {
-      const params = { items: 10000, features: 100, components: 3};    
-      const itemsProp = DG.Property.js('items', DG.TYPE.INT);
-      const featuresProp = DG.Property.js('features', DG.TYPE.INT);
-      const componentsProp = DG.Property.js('components', DG.TYPE.INT);      
-      ui.dialog({title:'Set'})
-        .add(ui.input.form(params, [itemsProp, featuresProp, componentsProp]))
-        .addButton('Run', async () => await demoPLS(params.items, params.features, params.components))
-        .show();      
-    }, {description: 'Random walk test dataframe of the given size is generated, and its multivariate analysis is performed.'})    
+    .step('Data', async () => {
+      grok.shell.addTableView(sourceCars);
+      view = grok.shell.getTableView(sourceCars.name);
+    }, {description: 'Each car has many features - patterns extraction is complicated.', delay: 0})
+    .step('Model', async () => {
+      dialog = ui.dialog({title:'Multivariate Analysis (PLS)'})
+        .add(ui.tableInput('Table', sourceCars))
+        .add(ui.columnsInput('Features', cars, features.toList, {available: undefined, checked: features.names()}))
+        .add(ui.columnInput('Names', cars, names, undefined))
+        .add(ui.columnInput('Predict', cars, predict, undefined))
+        .add(ui.intInput('Components', components, undefined))
+        .onOK(() => {
+          grok.shell.info('Multivariate analysis has been already performed.');
+        })
+        .show({x: 400, y: 140});
+    }, {description: 'Predict car price by its other features.', delay: 0})
+    .step('Regression coeffcicients', async () => 
+      {
+        dialog.close();
+        view.addViewer(regressionCoefficientsBarChart(features, plsOutput[1]))},
+      {description: 'The feature "diesel" affects the price the most.', delay: 0})
+    .step('Scores', async () => 
+      {view.addViewer(scoresScatterPlot(names, plsOutput[2], plsOutput[3]))}, 
+      {description: 'Similarities & dissimilarities: alfaromeo and mercedes are different.', delay: 0})
+    .step('Prediction', async () => 
+      {view.addViewer(predictedVersusReferenceScatterPlot(names, predict, plsOutput[0]))}, 
+      {description: 'Closer to the line means better price prediction.', delay: 0})
     .start();
 }
 

package/wasm/PLS/pls.cpp

@@ -12,177 +12,6 @@ using namespace Eigen;
 using pls::Float;
 using pls::Double;
 
-  /* Partial Least Square (PLS1).
-  predictorColumnsDataPtr - data from columns that are used for prediction
-  rowCount - number of rows
-  columnCount - number of columns
-  responseColumnDataPtr - data from column that is predicted, i.e. responce
-  componentsCount - number of components that extracted in PLS
-  predictionDataPtr - prediction obtained using PLS (its size is equal to the size of responce)
-  regressionCoefficients - coeffcient of linear regression that are computed (their size is eqaul to the number of columns)
-  */
-int pls::partialLeastSquare(Float * predictorColumnsDataPtr,
-	const int rowCount,
-	const int columnCount,
-	Float * responseColumnDataPtr,
-	const int componentsCount,
-	Float * predictionDataPtr,
-	Float * regressionCoefficients) noexcept
-{
-	// check correctness of arguments
-	if (componentsCount <= 0 || componentsCount > columnCount)
-		return UNCORRECT_ARGUMENTS_ERROR;
-
-	// Further, notation from the paper https://doi.org/10.1002/cem.2589 is used (see Algorithm 2).
-
-	// create matrix, which is associated with predictor data
-	Map < Matrix<Float, Dynamic, Dynamic, ColMajor>> D(predictorColumnsDataPtr, rowCount, columnCount);
-
-	// compute mean value of each column of D
-	Vector<Float, Dynamic> mu = D.colwise().mean();
-
-	// mean-centered version of D
-	Matrix<Float, Dynamic, Dynamic, ColMajor> X = D.rowwise() - mu.transpose();
-
-    // vector for standard deviations of X
-	Vector<Float, Dynamic> stdDevX(columnCount);
-	
-	Float rowCountSqrt = sqrt(static_cast<Float>(rowCount));
-
-    // normilizing X-columns
-	for (int i = 0; i < columnCount; i++)
-	{
-		stdDevX(i) = X.col(i).norm() / rowCountSqrt;
-		X.col(i) = X.col(i) / stdDevX(i);		
-	}	
-
-	// create a vector, which is associated with responce or predicted data 
-	Map<Vector<Float, Dynamic>> ySource(responseColumnDataPtr, rowCount);		
-
-    // mean value of the responce
-	Vector<Float, 1> meanY;
-	meanY(0) = ySource.mean();		
-
-    // mean-centered version of the responce
-	Vector<Float, Dynamic> y = ySource.rowwise() - meanY;
-
-    // standard deviation
-	Float stdDevY = sqrt(y.squaredNorm() / rowCount);
-
-    // normalizing
-	y /= stdDevY;	
-
-	// create a vector, which is associtated with regression coefficients
-	Map<Vector<Float, Dynamic>> b(regressionCoefficients, columnCount);
-
-	// create a vector, which is associated with prediction data 
-	Map<Vector<Float, Dynamic>> prediction(predictionDataPtr, rowCount);
-
-	// PLS1 algorithm routine
-
-	Matrix<Float, Dynamic, Dynamic, ColMajor> W(columnCount, componentsCount);
-
-	Matrix<Float, Dynamic, Dynamic, ColMajor> P(columnCount, componentsCount);
-
-	Matrix<Float, Dynamic, Dynamic, ColMajor> T(rowCount, componentsCount);
-
-	Vector<Float, Dynamic> normTau(componentsCount);
-
-	Vector<Float, Dynamic> q(componentsCount);
-
-	Vector<Float, Dynamic> normV(componentsCount);
-
-	// PLS1 algorithm: see Algorithm 2 in https://doi.org/10.1002/cem.2589
-
-	Vector<Float, Dynamic> w = (X.transpose() * y);
-
-	normV(0) = w.norm();
-
-	// prevent division by zero
-	if (normV(0) == static_cast<Float>(0))
-		return METHOD_ERROR;
-
-	w = w / normV(0);
-
-	W.col(0) = w;
-
-	Vector<Float, Dynamic> t = X * w;
-
-	normTau(0) = t.norm();
-
-	// prevent division by zero
-	if (normTau(0) == static_cast<Float>(0))
-		return METHOD_ERROR;
-
-	t = t / normTau(0);
-
-	T.col(0) = t;
-
-	Vector<Float, Dynamic> p = X.transpose() * t;
-
-	P.col(0) = p;
-
-	q(0) = t.transpose() * y;
-
-	for (int a = 1; a < componentsCount; a++)
-	{
-		w = normV(a - 1) * (w - p / normTau(a - 1));
-
-		normV(a) = w.norm();
-
-		// prevent division by zero
-		if (normV(a) == static_cast<Float>(0))
-			return METHOD_ERROR;
-
-		w = w / normV(a);
-
-		W.col(a) = w;
-
-		t = X * w;
-
-		t = t - T.leftCols(a) * (T.leftCols(a).transpose() * t);
-
-		normTau(a) = t.norm();
-
-		// prevent division by zero
-		if (normTau(a) == static_cast<Float>(0))
-			return METHOD_ERROR;
-
-		t = t / normTau(a);
-
-		T.col(a) = t;
-
-		p = X.transpose() * t;
-
-		P.col(a) = p;
-
-		q(a) = t.transpose() * y;
-	} // for a	
-
-	  // compute coefficients of regression
-	Matrix<Float, Dynamic, Dynamic> H = P.transpose() * W;
-
-	// chech existence of inverse matrix
-	if (H.determinant() == static_cast<Float>(0))
-		return METHOD_ERROR;
-
-	b = W * H.inverse() * q;
-
-	for (int i = 0; i < columnCount; i++)
-		b(i) *= stdDevY / stdDevX(i);   
-	
-	// TODO: to discuss a constant term of the regression
-	// a constant term
-	//Vector<Float, 1> shift;
-	//shift(0) = ySource(0) - D.row(0) * b;
-	//q(0) - P.col(0).transpose().dot(b);	
-	//prediction = (D * b).rowwise() + shift;
-
-	prediction = D * b;	
-
-	return NO_ERROR;
-} // partialLeastSquare
-
 /* Partial Least Square (PLS1) - extended version: scores data is provided.
       predictorColumnsDataPtr - data from columns that are used for prediction (X)
       rowCount - number of rows
@@ -358,7 +187,7 @@ int pls::partialLeastSquareExtended(Float * predictorColumnsDataPtr,
 	// compute coefficients of regression
 	Matrix<Float, Dynamic, Dynamic> H = P.transpose() * W;
 
-	// chech existence of inverse matrix
+	// check existence of inverse matrix
 	if (H.determinant() == static_cast<Float>(0))
 		return METHOD_ERROR;
 
@@ -370,7 +199,7 @@ int pls::partialLeastSquareExtended(Float * predictorColumnsDataPtr,
 		b(i) *= stdDevY / stdDevX(i);
 
 	// compute predictions
-	prediction = D * b;	
+	prediction = D * b;
 
 	// Remove the following comments in order to print and verify results
 	//cout << "\nW_star:\n" << Wstar << endl;	

package/wasm/callWasm.js

@@ -397,9 +397,6 @@ let Return = {
 
 // The main tool that combines all together
 export function callWasm(module, funcName, inputs) {
-
-    let start = new Date().getTime();
-
     // get specification of exported C/C++-function
     let funcSpecification = module[funcName];
 
@@ -449,18 +446,12 @@ export function callWasm(module, funcName, inputs) {
     // CALL EXPORTED CPP-FUNCTION
     let callResult = cppFuncWrapper(module, funcName, 'num', cppFuncInput);
 
-    console.log('C++-function call result: ' + callResult);
-
     // store result that is returned by exported cpp-function
     args._callResult = Param.num(callResult);
 
     // create output
     let output = funcSpecification.output;
 
-    let finish = new Date().getTime();
-
-    console.log(`Time for C/C++-function is ${finish - start} ms.`)
-
     // if a single object must be returned
     if(output['type'] != 'objects')
       return Return[output['type']](args[output['source']].data);

package/src/demos.ts

@@ -1,38 +0,0 @@
-/* Do not change these import lines to match external modules in webpack configuration */
-import * as grok from 'datagrok-api/grok';
-import * as ui from 'datagrok-api/ui';
-import * as DG from 'datagrok-api/dg';
-
-import {computePLS} from './EDAtools';
-import {addPLSvisualization} from './EDAui';
-
-// Demo multivariate analysis (PLS)
-export async function demoPLS(rowCount: number, colCount: number, componentsCount: number): Promise<void> {
-  // check inputs
-  if ((rowCount <= 0) || (colCount <= 0) || (componentsCount <= 0) || (componentsCount > colCount)) {
-    const bal = new DG.Balloon;
-    bal.error('Incorrect inputs.');
-    return;
-  }
-
-  // further, custom interface is provided
-  
-  const PREDICT = 'Reference';
-  
-  const bigDemoTable = grok.data.testData('random walk', rowCount, colCount);   
-  bigDemoTable.name = `${rowCount} x ${colCount}`;
-  
-  for (const col of bigDemoTable.columns)
-    col.name = 'Feature ' + col.name;
-  bigDemoTable.columns.byIndex(0).name = PREDICT;
-  
-  grok.shell.addTableView(bigDemoTable);
-  let predict = bigDemoTable.columns.byName(PREDICT);
-  let features = bigDemoTable.columns.remove(PREDICT);      
-  
-  const plsResults = await computePLS(bigDemoTable, features, predict, componentsCount);
-  
-  addPLSvisualization(bigDemoTable, features, predict, plsResults);
-  
-  bigDemoTable.columns.add(predict);
-}