@datagrok/eda 1.1.8 → 1.1.10

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@datagrok/eda",
   "friendlyName": "EDA",
-  "version": "1.1.8",
+  "version": "1.1.10",
   "description": "Exploratory Data Analysis Tools",
   "dependencies": {
-    "@datagrok-libraries/ml": "^6.3.70",
+    "@datagrok-libraries/ml": "^6.4.1",
     "@datagrok-libraries/tutorials": "^1.3.6",
     "@datagrok-libraries/utils": "^4.1.4",
     "@datagrok-libraries/math": "^1.0.7",
@@ -13,7 +13,9 @@
     "datagrok-api": "^1.16.0",
     "dayjs": "^1.11.9",
     "jstat": "^1.9.6",
-    "umap-js": "^1.3.3"
+    "umap-js": "^1.3.3",
+    "worker-loader": "latest",
+    "source-map-loader": "^4.0.1"
   },
   "author": {
     "name": "Viktor Makarichev",
@@ -23,7 +25,11 @@
     "ts-loader": "latest",
     "typescript": "latest",
     "webpack": "latest",
-    "webpack-cli": "latest"
+    "webpack-cli": "latest",
+    "@typescript-eslint/eslint-plugin": "^5.32.0",
+    "@typescript-eslint/parser": "^5.32.0",
+    "eslint": "^8.21.0",
+    "eslint-config-google": "^0.14.0"
   },
   "scripts": {
     "link-all": "npm link datagrok-api @datagrok-libraries/utils @datagrok-libraries/tutorials",

package/src/data-generators.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-len */
 // Test data generation tools
 
 /* Do not change these import lines to match external modules in webpack configuration */
@@ -21,7 +22,7 @@ export function carsDataframe(): DG.DataFrame {
       DG.Column.fromInt32Array('diesel', new Int32Array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1])),
       DG.Column.fromInt32Array('turbo', new Int32Array([0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1])),
       DG.Column.fromInt32Array('two.doors', new Int32Array([1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0])),
-      DG.Column.fromInt32Array('hatchback', new Int32Array([1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0])),      
+      DG.Column.fromInt32Array('hatchback', new Int32Array([1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0])),
       DG.Column.fromFloat32Array('wheel.base', new Float32Array([94.5, 105.80000305175781, 101.19999694824219, 94.5, 93.69999694824219, 93.69999694824219, 93.69999694824219, 96.5, 94.30000305175781, 113, 93.0999984741211, 115.5999984741211, 102.69999694824219, 93.69999694824219, 94.5, 94.5, 93.69999694824219, 114.19999694824219, 89.5, 99.0999984741211, 97.19999694824219, 95.69999694824219, 95.69999694824219, 98.4000015258789, 102.4000015258789, 97.30000305175781, 100.4000015258789, 104.30000305175781, 109.0999984741211, 109.0999984741211])),
       DG.Column.fromFloat32Array('length', new Float32Array([171.1999969482422, 192.6999969482422, 176.8000030517578, 158.8000030517578, 157.3000030517578, 157.3000030517578, 150, 175.39999389648438, 170.6999969482422, 199.60000610351562, 166.8000030517578, 202.60000610351562, 178.39999389648438, 157.3000030517578, 170.1999969482422, 165.3000030517578, 157.3000030517578, 198.89999389648438, 168.89999389648438, 186.60000610351562, 172, 158.6999969482422, 166.3000030517578, 176.1999969482422, 175.60000610351562, 171.6999969482422, 180.1999969482422, 188.8000030517578, 188.8000030517578, 188.8000030517578])),
       DG.Column.fromFloat32Array('width', new Float32Array([65.5, 71.4000015258789, 64.80000305175781, 63.599998474121094, 63.79999923706055, 63.79999923706055, 64, 65.19999694824219, 61.79999923706055, 69.5999984741211, 64.19999694824219, 71.69999694824219, 68, 64.4000015258789, 63.79999923706055, 63.79999923706055, 63.79999923706055, 68.4000015258789, 65, 66.5, 65.4000015258789, 63.599998474121094, 64.4000015258789, 65.5999984741211, 66.5, 65.5, 66.9000015258789, 67.19999694824219, 68.80000305175781, 68.9000015258789])),
@@ -38,36 +39,35 @@ export function carsDataframe(): DG.DataFrame {
 } // carsDataframe
 
 // Generate dataset for testing binary classifiers
-export async function testDataForBinaryClassification(kernel: number, kernelParams: Array<number>, 
-  name: string, samplesCount: number, featuresCount: number, min: number, 
+export async function testDataForBinaryClassification(kernel: number, kernelParams: Array<number>,
+  name: string, samplesCount: number, featuresCount: number, min: number,
   max: number, violatorsPercentage: number): Promise<DG.DataFrame> {
-
   // check inputs
   checkGeneratorSVMinputs(samplesCount, featuresCount, min, max, violatorsPercentage);
-  
+
   // kernel params column
   const kernelParamsCol = DG.Column.fromList('double', 'kernelParams', kernelParams);
-  
-  // CALL WASM-COMPUTATIONS  
+
+  // CALL WASM-COMPUTATIONS
   let _output: any;
-  let _promise = _generateDatasetInWebWorker(kernel, kernelParamsCol, 
+  const _promise = _generateDatasetInWebWorker(kernel, kernelParamsCol,
     samplesCount, featuresCount, min, max, violatorsPercentage);
 
   await _promise.then(
-    _result => { _output = _result; },
-    _error => {  throw new Error (`Error: ${_error}`); }
+    (_result) => {_output = _result;},
+    (_error) => {throw new Error(`Error: ${_error}`);},
   );
-  
+
   // Rename labels column
   _output[SVM_GEN_LABELS_INDEX].name = SVM_LABEL_NAME;
-  
+
   // Rename feature columns
   for (const col of _output[SVM_GEN_FEATURES_INDEX])
     col.name = SVM_FEATURE_NAME + col.name;
 
   // Create dataframe
   const df = DG.DataFrame.fromColumns(_output[SVM_GEN_FEATURES_INDEX]);
-  df.name = name;  
+  df.name = name;
   df.columns.add(_output[SVM_GEN_LABELS_INDEX]);
 
   return df;

package/src/eda-tools.ts

@@ -4,30 +4,27 @@ import * as grok from 'datagrok-api/grok';
 import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
 
-import {DimensionalityReducer} from '@datagrok-libraries/ml/src/reduce-dimensionality';
-import {VectorMetricsNames} from '@datagrok-libraries/ml/src/typed-metrics';
-
 import {_principalComponentAnalysisInWebWorker,
   _partialLeastSquareRegressionInWebWorker} from '../wasm/EDAAPI';
 
-import {checkWasmDimensionReducerInputs, checkUMAPinputs, checkTSNEinputs, checkSPEinputs,
+import {checkWasmDimensionReducerInputs, checkUMAPinputs, checkTSNEinputs,
   getRowsOfNumericalColumnns} from './utils';
 
 // Principal components analysis (PCA)
 export async function computePCA(table: DG.DataFrame, features: DG.ColumnList, components: number,
-  center: boolean, scale: boolean): Promise<DG.DataFrame> 
-{
+  center: boolean, scale: boolean): Promise<DG.DataFrame> {
   checkWasmDimensionReducerInputs(features, components);
 
   const centerNum = center ? 1 : 0;
   const scaleNum = scale ? 1 : 0;
 
   return await _principalComponentAnalysisInWebWorker(table, features, components, centerNum, scaleNum);
-} 
+}
 
 // Partial least square regression (PLS)
-export async function computePLS(table: DG.DataFrame, features: DG.ColumnList, predict: DG.Column, components: number): Promise<any> 
-{
+export async function computePLS(
+  table: DG.DataFrame, features: DG.ColumnList, predict: DG.Column, components: number,
+): Promise<any> {
   // Inputs are checked in the same manner as in PCA, since the same computations are applied.
   checkWasmDimensionReducerInputs(features, components);
 
@@ -36,10 +33,9 @@ export async function computePLS(table: DG.DataFrame, features: DG.ColumnList, p
 
 // Uniform Manifold Approximation and Projection (UMAP)
 export async function computeUMAP(features: DG.ColumnList, components: number, epochs: number,
-  neighbors: number, minDist: number, spread: number): Promise<DG.DataFrame> 
-{
+  neighbors: number, minDist: number, spread: number): Promise<DG.DataFrame> {
   // check inputs
-  checkUMAPinputs(features, components, epochs, neighbors, minDist, spread);  
+  checkUMAPinputs(features, components, epochs, neighbors, minDist, spread);
 
   // get row-by-row data
   const data = getRowsOfNumericalColumnns(features);
@@ -47,27 +43,28 @@ export async function computeUMAP(features: DG.ColumnList, components: number, e
   let workerOutput: any;
 
   // UMAP in webworker
-  let promise = new Promise((resolve, reject) => {
+  const promise = new Promise((resolve, _reject) => {
     const worker = new Worker(new URL('workers/umap-worker.ts', import.meta.url));
-    
+
     worker.postMessage({
-      data: data, 
+      data: data,
       options: {
         nComponents: components,
         nEpochs: epochs,
         nNeighbors: neighbors,
         minDist: minDist,
-        spread: spread
-    }});
+        spread: spread,
+      }});
 
     worker.onmessage = function(e) {
       worker.terminate();
-      resolve(e.data.embeddings);    
-  }}); 
-  
+      resolve(e.data.embeddings);
+    };
+  });
+
   await promise.then(
-    result => { workerOutput = result; },
-    error => { throw new Error ('applying UMAP fails.'); }
+    (result) => {workerOutput = result;},
+    (_error) => {throw new Error('applying UMAP fails.');},
   );
 
   const embeddings = workerOutput as number[][];
@@ -77,22 +74,22 @@ export async function computeUMAP(features: DG.ColumnList, components: number, e
   // Create output
 
   // columns data
-  const umapColumnsData = range.map(_ => new Float32Array(rowCount));  
+  const umapColumnsData = range.map((_) => new Float32Array(rowCount));
 
   // perform transponation
-  for (let i = 0; i < rowCount; ++i)
+  for (let i = 0; i < rowCount; ++i) {
     for (let j = 0; j < components; ++j)
       umapColumnsData[j][i] = embeddings[i][j];
+  }
 
-  return DG.DataFrame.fromColumns(range.map(i => 
-    DG.Column.fromFloat32Array('UMAP' + i.toString(), umapColumnsData[i])
+  return DG.DataFrame.fromColumns(range.map((i) =>
+    DG.Column.fromFloat32Array('UMAP' + i.toString(), umapColumnsData[i]),
   ));
 } // computeUMAP
 
 // t-distributed stochastic neighbor embedding (t-SNE)
-export async function computeTSNE(features: DG.ColumnList, components: number, 
-  learningRate: number, perplexity: number, iterations: number): Promise<DG.DataFrame> 
-{
+export async function computeTSNE(features: DG.ColumnList, components: number,
+  learningRate: number, perplexity: number, iterations: number): Promise<DG.DataFrame> {
   // check inputs
   checkTSNEinputs(features, components, learningRate, perplexity, iterations);
 
@@ -102,26 +99,27 @@ export async function computeTSNE(features: DG.ColumnList, components: number,
   let workerOutput: any;
 
   // t-SNE in webworker
-  let promise = new Promise((resolve, reject) => {
+  const promise = new Promise((resolve, _reject) => {
     const worker = new Worker(new URL('workers/tsne-worker.ts', import.meta.url));
-    
+
     worker.postMessage({
-      data: data, 
+      data: data,
       options: {
         learningRate: learningRate,
         perplexity: perplexity,
         components: components,
-        iterations: iterations
-    }});
+        iterations: iterations,
+      }});
 
     worker.onmessage = function(e) {
       worker.terminate();
-      resolve(e.data.embeddings);    
-  }}); 
-  
+      resolve(e.data.embeddings);
+    };
+  });
+
   await promise.then(
-    result => { workerOutput = result; },
-    error => { throw new Error ('applying t-SNE fails.'); }
+    (result) => {workerOutput = result;},
+    (_error) => {throw new Error('applying t-SNE fails.');},
   );
 
   const embeddings = workerOutput as any[];
@@ -132,54 +130,15 @@ export async function computeTSNE(features: DG.ColumnList, components: number,
   // Create output
 
   // columns data
-  const umapColumnsData = range.map(_ => new Float32Array(rowCount));  
+  const umapColumnsData = range.map((_) => new Float32Array(rowCount));
 
   // perform transponation
-  for (let i = 0; i < rowCount; ++i)
+  for (let i = 0; i < rowCount; ++i) {
     for (let j = 0; j < components; ++j)
       umapColumnsData[j][i] = embeddings[i][j];
+  }
 
-  return DG.DataFrame.fromColumns(range.map(i => 
-    DG.Column.fromFloat32Array('tSNE' + i.toString(), umapColumnsData[i])
+  return DG.DataFrame.fromColumns(range.map((i) =>
+    DG.Column.fromFloat32Array('tSNE' + i.toString(), umapColumnsData[i]),
   ));
 } // computeTSNE
-
-// Stochastic proximity embedding (SPE)
-export async function computeSPE(features: DG.ColumnList, dimension: number,
-  steps: number, cycles: number, cutoff: number, lambda: number): Promise<DG.DataFrame> 
-{
-  // check inputs
-  checkSPEinputs(features, dimension, steps, cycles, cutoff, lambda);
-
-  // get row-by-row data
-  const data = getRowsOfNumericalColumnns(features);
-
-  // SPE reducer
-  const spe = new DimensionalityReducer(data, 'SPE', VectorMetricsNames.Euclidean, {
-    dimension: dimension,
-    steps: steps,
-    cycles: cycles,
-    cutoff: cutoff,
-    lambda: lambda
-  });
-
-  // compute embeddings
-  const embeddings = (await spe.transform(false, false)).embedding;
-  
-  const rowCount = embeddings.length;
-  const range = [...Array(dimension).keys()];
-  
-  // Create output
-  
-  // columns data
-  const umapColumnsData = range.map(_ => new Float32Array(rowCount));  
-  
-  // perform transponation
-  for (let i = 0; i < rowCount; ++i)
-    for (let j = 0; j < dimension; ++j)
-      umapColumnsData[j][i] = embeddings[i][j];
-  
-  return DG.DataFrame.fromColumns(range.map(i => 
-    DG.Column.fromFloat32Array('SPE' + i.toString(), umapColumnsData[i])
-  ));
-} // computeSPE

package/src/eda-ui.ts

@@ -5,8 +5,8 @@ import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
 
 // Rename PCA columns
-export function renamePCAcolumns(pcaTable: DG.DataFrame): DG.DataFrame {  
-  for (const col of pcaTable.columns.toList()) 
+export function renamePCAcolumns(pcaTable: DG.DataFrame): DG.DataFrame {
+  for (const col of pcaTable.columns.toList())
     col.name = 'PCA' + col.name;
 
   return pcaTable;
@@ -19,116 +19,123 @@ export function addPrefixToEachColumnName(prefix: string, columns: DG.ColumnList
 }
 
 // Predicted vs Reference scatter plot
-export function predictedVersusReferenceScatterPlot(samplesNames: DG.Column, reference: DG.Column, prediction: DG.Column): DG.Viewer {  
+export function predictedVersusReferenceScatterPlot(
+  samplesNames: DG.Column, reference: DG.Column, prediction: DG.Column,
+): DG.Viewer {
   prediction.name = reference.name + '(predicted)';
- 
-  let dfReferencePrediction = DG.DataFrame.fromColumns([samplesNames, reference, prediction]);
+
+  const dfReferencePrediction = DG.DataFrame.fromColumns([samplesNames, reference, prediction]);
   dfReferencePrediction.name = 'Reference vs. Predicted';
-   
-  return DG.Viewer.scatterPlot(dfReferencePrediction, 
-    { title: dfReferencePrediction.name,
+
+  return DG.Viewer.scatterPlot(dfReferencePrediction,
+    {title: dfReferencePrediction.name,
       x: reference.name,
       y: prediction.name,
       showRegressionLine: true,
       markerType: 'circle',
-      labels: samplesNames.name
-     });  
+      labels: samplesNames.name,
+    });
 }
 
 // Regression Coefficients Bar Chart
-export function regressionCoefficientsBarChart(features: DG.ColumnList, regressionCoeffs: DG.Column): DG.Viewer {    regressionCoeffs.name = 'regression coefficient';
- 
-  let namesOfPredictors = [];
+export function regressionCoefficientsBarChart(features: DG.ColumnList, regressionCoeffs: DG.Column): DG.Viewer {
+  regressionCoeffs.name = 'regression coefficient';
+
+  const namesOfPredictors = [];
   for (const col of features)
-    namesOfPredictors.push(col.name); 
-  
-  let predictorNamesColumn = DG.Column.fromStrings('feature', namesOfPredictors);  
- 
-  let dfRegrCoefs = DG.DataFrame.fromColumns([predictorNamesColumn, regressionCoeffs]);
+    namesOfPredictors.push(col.name);
+
+  const predictorNamesColumn = DG.Column.fromStrings('feature', namesOfPredictors);
+
+  const dfRegrCoefs = DG.DataFrame.fromColumns([predictorNamesColumn, regressionCoeffs]);
   dfRegrCoefs.name = 'Regression Coefficients';
 
-  return DG.Viewer.barChart(dfRegrCoefs, 
-    {title: dfRegrCoefs.name, split: 'feature', 
-     value: 'regression coefficient', valueAggrType: 'avg'}); 
+  return DG.Viewer.barChart(dfRegrCoefs,
+    {title: dfRegrCoefs.name, split: 'feature',
+      value: 'regression coefficient', valueAggrType: 'avg'});
 }
 
 // Scores Scatter Plot
-export function scoresScatterPlot(samplesNames: DG.Column, xScores: Array<DG.Column>, yScores: Array<DG.Column>): DG.Viewer {
-  
-  let scoresColumns = [samplesNames];
+export function scoresScatterPlot(
+  samplesNames: DG.Column, xScores: Array<DG.Column>, yScores: Array<DG.Column>,
+): DG.Viewer {
+  const scoresColumns = [samplesNames];
 
   for (let i = 0; i < xScores.length; i++) {
     xScores[i].name = `x.score.t${i+1}`;
     scoresColumns.push(xScores[i]);
-  } 
-  
+  }
+
   for (let i = 0; i < yScores.length; i++) {
     yScores[i].name = `y.score.u${i+1}`;
     scoresColumns.push(yScores[i]);
-  }  
- 
-  let scores = DG.DataFrame.fromColumns(scoresColumns);
+  }
+
+  const scores = DG.DataFrame.fromColumns(scoresColumns);
   scores.name = 'Scores';
   //grok.shell.addTableView(scores);
 
   const index = xScores.length > 1 ? 1 : 0;
- 
-  return DG.Viewer.scatterPlot(scores, 
-    { title: scores.name,
+
+  return DG.Viewer.scatterPlot(scores,
+    {title: scores.name,
       x: xScores[0].name,
-      y: xScores[index].name,      
+      y: xScores[index].name,
       markerType: 'circle',
-      labels: samplesNames.name
-     });
+      labels: samplesNames.name,
+    });
 }
 
 // Loading Scatter Plot
 export function loadingScatterPlot(features: DG.ColumnList, xLoadings: Array<DG.Column>): DG.Viewer {
-  let loadingCols = [];
- 
-  let loadingLabels = [];
-  for (let col of features)
+  const loadingCols = [];
+
+  const loadingLabels = [];
+  for (const col of features)
     loadingLabels.push(col.name);
- 
-  loadingCols.push(DG.Column.fromStrings('labels', loadingLabels)); 
-  
+
+  loadingCols.push(DG.Column.fromStrings('labels', loadingLabels));
+
   for (let i = 0; i < xLoadings.length; i++) {
     xLoadings[i].name = `x.loading.p${i+1}`;
     loadingCols.push(xLoadings[i]);
   }
- 
-  let dfLoadings = DG.DataFrame.fromColumns(loadingCols);
+
+  const dfLoadings = DG.DataFrame.fromColumns(loadingCols);
   dfLoadings.name = 'Loadings';
-   
-  return DG.Viewer.scatterPlot(dfLoadings, 
-    { title: dfLoadings.name,
+
+  return DG.Viewer.scatterPlot(dfLoadings,
+    {title: dfLoadings.name,
       x: xLoadings[0].name,
-      y: xLoadings[xLoadings.length - 1].name,      
+      y: xLoadings[xLoadings.length - 1].name,
       markerType: 'circle',
-      labels: 'labels'
-     });
+      labels: 'labels',
+    });
 }
 
 // Add PLS visualization
-export function addPLSvisualization(table: DG.DataFrame, samplesNames: DG.Column, features: DG.ColumnList, predict: DG.Column, plsOutput: any): void {
-
+export function addPLSvisualization(
+  table: DG.DataFrame, samplesNames: DG.Column, features: DG.ColumnList, predict: DG.Column, plsOutput: any,
+): void {
   const view = grok.shell.getTableView(table.name);
- 
+
   // 1. Predicted vs Reference scatter plot
   view.addViewer(predictedVersusReferenceScatterPlot(samplesNames, predict, plsOutput[0]));
-  
+
   // 2. Regression Coefficients Bar Chart
-  view.addViewer(regressionCoefficientsBarChart(features, plsOutput[1]));   
- 
+  view.addViewer(regressionCoefficientsBarChart(features, plsOutput[1]));
+
   // 3. Loading Scatter Plot
-  view.addViewer(loadingScatterPlot(features, plsOutput[4]));  
+  view.addViewer(loadingScatterPlot(features, plsOutput[4]));
 
   // 4. Scores Scatter Plot
   view.addViewer(scoresScatterPlot(samplesNames, plsOutput[2], plsOutput[3]));
 }
 
 // Add one-way ANOVA results
-export function addOneWayAnovaVizualization(table: DG.DataFrame, factors: DG.Column, values: DG.Column, anova: DG.DataFrame) {
+export function addOneWayAnovaVizualization(
+  table: DG.DataFrame, factors: DG.Column, values: DG.Column, anova: DG.DataFrame,
+) {
   const view = grok.shell.getTableView(table.name);
   view.addViewer(DG.Viewer.boxPlot(DG.DataFrame.fromColumns([factors, values])));
   view.addViewer(DG.Viewer.grid(anova));

package/src/package-test.ts

@@ -1,7 +1,7 @@
-import * as DG from "datagrok-api/dg";
+import * as DG from 'datagrok-api/dg';
 import {runTests, tests, TestContext} from '@datagrok-libraries/utils/src/test';
 
-export let _package = new DG.Package();
+export const _package = new DG.Package();
 export {tests};
 
 //name: test