@datagrok/eda 1.1.2 → 1.1.3

package/package.json

@@ -1,24 +1,27 @@
 {
   "name": "@datagrok/eda",
   "friendlyName": "EDA",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "Exploratory Data Analysis Tools",
   "dependencies": {
-    "datagrok-api": "latest",
-    "cash-dom": "latest",
-    "dayjs": "latest",
-    "@datagrok-libraries/utils": "latest",
-    "@datagrok-libraries/tutorials": "^1.3.6"
+    "@datagrok-libraries/ml": "^6.3.39",
+    "@datagrok-libraries/tutorials": "^1.3.6",
+    "@datagrok-libraries/utils": "^4.1.4",
+    "@keckelt/tsne": "^1.0.2",
+    "cash-dom": "^8.1.1",
+    "datagrok-api": "^1.16.0",
+    "dayjs": "^1.11.9",
+    "umap-js": "^1.3.3"
   },
   "author": {
     "name": "Viktor Makarichev",
     "email": "vmakarichev@datagrok.ai"
   },
   "devDependencies": {
-    "webpack": "latest",
-    "webpack-cli": "latest",
     "ts-loader": "latest",
-    "typescript": "latest"
+    "typescript": "latest",
+    "webpack": "latest",
+    "webpack-cli": "latest"
   },
   "scripts": {
     "link-all": "npm link datagrok-api @datagrok-libraries/utils @datagrok-libraries/tutorials",

package/src/eda-tools.ts

@@ -0,0 +1,185 @@
+// Exploratory data analysis (EDA) tools
+
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+
+import {DimensionalityReducer} from '@datagrok-libraries/ml/src/reduce-dimensionality';
+import {VectorMetricsNames} from '@datagrok-libraries/ml/src/typed-metrics';
+
+import {_principalComponentAnalysisInWebWorker,
+  _partialLeastSquareRegressionInWebWorker} from '../wasm/EDAAPI';
+
+import {checkWasmDimensionReducerInputs, checkUMAPinputs, checkTSNEinputs, checkSPEinputs,
+  getRowsOfNumericalColumnns} from './utils';
+
+// Principal components analysis (PCA)
+export async function computePCA(table: DG.DataFrame, features: DG.ColumnList, components: number,
+  center: boolean, scale: boolean): Promise<DG.DataFrame> 
+{
+  checkWasmDimensionReducerInputs(features, components);
+
+  const centerNum = center ? 1 : 0;
+  const scaleNum = scale ? 1 : 0;
+
+  return await _principalComponentAnalysisInWebWorker(table, features, components, centerNum, scaleNum);
+} 
+
+// Partial least square regression (PLS)
+export async function computePLS(table: DG.DataFrame, features: DG.ColumnList, predict: DG.Column, components: number): Promise<any> 
+{
+  // Inputs are checked in the same manner as in PCA, since the same computations are applied.
+  checkWasmDimensionReducerInputs(features, components);
+
+  return await _partialLeastSquareRegressionInWebWorker(table, features, predict, components);
+}
+
+// Uniform Manifold Approximation and Projection (UMAP)
+export async function computeUMAP(features: DG.ColumnList, components: number, epochs: number,
+  neighbors: number, minDist: number, spread: number): Promise<DG.DataFrame> 
+{
+  // check inputs
+  checkUMAPinputs(features, components, epochs, neighbors, minDist, spread);  
+
+  // get row-by-row data
+  const data = getRowsOfNumericalColumnns(features);
+
+  let workerOutput: any;
+
+  // UMAP in webworker
+  let promise = new Promise((resolve, reject) => {
+    const worker = new Worker(new URL('workers/umap-worker.ts', import.meta.url));
+    
+    worker.postMessage({
+      data: data, 
+      options: {
+        nComponents: components,
+        nEpochs: epochs,
+        nNeighbors: neighbors,
+        minDist: minDist,
+        spread: spread
+    }});
+
+    worker.onmessage = function(e) {
+      worker.terminate();
+      resolve(e.data.embeddings);    
+  }}); 
+  
+  await promise.then(
+    result => { workerOutput = result; },
+    error => { throw new Error ('applying UMAP fails.'); }
+  );
+
+  const embeddings = workerOutput as number[][];
+  const rowCount = embeddings.length;
+  const range = [...Array(components).keys()];
+
+  // Create output
+
+  // columns data
+  const umapColumnsData = range.map(_ => new Float32Array(rowCount));  
+
+  // perform transponation
+  for (let i = 0; i < rowCount; ++i)
+    for (let j = 0; j < components; ++j)
+      umapColumnsData[j][i] = embeddings[i][j];
+
+  return DG.DataFrame.fromColumns(range.map(i => 
+    DG.Column.fromFloat32Array('UMAP' + i.toString(), umapColumnsData[i])
+  ));
+} // computeUMAP
+
+// t-distributed stochastic neighbor embedding (t-SNE)
+export async function computeTSNE(features: DG.ColumnList, components: number, 
+  learningRate: number, perplexity: number, iterations: number): Promise<DG.DataFrame> 
+{
+  // check inputs
+  checkTSNEinputs(features, components, learningRate, perplexity, iterations);
+
+  // get row-by-row data
+  const data = getRowsOfNumericalColumnns(features);
+
+  let workerOutput: any;
+
+  // t-SNE in webworker
+  let promise = new Promise((resolve, reject) => {
+    const worker = new Worker(new URL('workers/tsne-worker.ts', import.meta.url));
+    
+    worker.postMessage({
+      data: data, 
+      options: {
+        learningRate: learningRate,
+        perplexity: perplexity,
+        components: components,
+        iterations: iterations
+    }});
+
+    worker.onmessage = function(e) {
+      worker.terminate();
+      resolve(e.data.embeddings);    
+  }}); 
+  
+  await promise.then(
+    result => { workerOutput = result; },
+    error => { throw new Error ('applying t-SNE fails.'); }
+  );
+
+  const embeddings = workerOutput as any[];
+
+  const rowCount = embeddings.length;
+  const range = [...Array(components).keys()];
+
+  // Create output
+
+  // columns data
+  const umapColumnsData = range.map(_ => new Float32Array(rowCount));  
+
+  // perform transponation
+  for (let i = 0; i < rowCount; ++i)
+    for (let j = 0; j < components; ++j)
+      umapColumnsData[j][i] = embeddings[i][j];
+
+  return DG.DataFrame.fromColumns(range.map(i => 
+    DG.Column.fromFloat32Array('tSNE' + i.toString(), umapColumnsData[i])
+  ));
+} // computeTSNE
+
+// Stochastic proximity embedding (SPE)
+export async function computeSPE(features: DG.ColumnList, dimension: number,
+  steps: number, cycles: number, cutoff: number, lambda: number): Promise<DG.DataFrame> 
+{
+  // check inputs
+  checkSPEinputs(features, dimension, steps, cycles, cutoff, lambda);
+
+  // get row-by-row data
+  const data = getRowsOfNumericalColumnns(features);
+
+  // SPE reducer
+  const spe = new DimensionalityReducer(data, 'SPE', VectorMetricsNames.Euclidean, {
+    dimension: dimension,
+    steps: steps,
+    cycles: cycles,
+    cutoff: cutoff,
+    lambda: lambda
+  });
+
+  // compute embeddings
+  const embeddings = (await spe.transform(false, false)).embedding;
+  
+  const rowCount = embeddings.length;
+  const range = [...Array(dimension).keys()];
+  
+  // Create output
+  
+  // columns data
+  const umapColumnsData = range.map(_ => new Float32Array(rowCount));  
+  
+  // perform transponation
+  for (let i = 0; i < rowCount; ++i)
+    for (let j = 0; j < dimension; ++j)
+      umapColumnsData[j][i] = embeddings[i][j];
+  
+  return DG.DataFrame.fromColumns(range.map(i => 
+    DG.Column.fromFloat32Array('SPE' + i.toString(), umapColumnsData[i])
+  ));
+} // computeSPE

package/src/{EDAui.ts → eda-ui.ts}

@@ -12,6 +12,12 @@ export function renamePCAcolumns(pcaTable: DG.DataFrame): DG.DataFrame {
   return pcaTable;
 }
 
+// Adds prefix to each column name
+export function addPrefixToEachColumnName(prefix: string, columns: DG.ColumnList): void {
+  for (const col of columns.toList())
+    col.name = prefix + col.name;
+}
+
 // Predicted vs Reference scatter plot
 export function predictedVersusReferenceScatterPlot(samplesNames: DG.Column, reference: DG.Column, prediction: DG.Column): DG.Viewer {  
   prediction.name = reference.name + '(predicted)';

package/src/package.ts

@@ -6,10 +6,10 @@ import * as DG from 'datagrok-api/dg';
 import {DemoScript} from '@datagrok-libraries/tutorials/src/demo-script';
 
 import {_initEDAAPI} from '../wasm/EDAAPI';
-import {computePCA, computePLS} from './EDAtools';
-import {renamePCAcolumns, addPLSvisualization, regressionCoefficientsBarChart, 
-  scoresScatterPlot, predictedVersusReferenceScatterPlot} from './EDAui';
-import {carsDataframe, testDataForBinaryClassification} from './dataGenerators';
+import {computePCA, computePLS, computeUMAP, computeTSNE, computeSPE} from './eda-tools';
+import {addPrefixToEachColumnName, addPLSvisualization, regressionCoefficientsBarChart, 
+  scoresScatterPlot, predictedVersusReferenceScatterPlot} from './eda-ui';
+import {carsDataframe, testDataForBinaryClassification} from './data-generators';
 import {LINEAR, RBF, POLYNOMIAL, SIGMOID, 
   getTrainedModel, getPrediction, showTrainReport, getPackedModel} from './svm';
 
@@ -25,19 +25,71 @@ export async function init(): Promise<void> {
   await _initEDAAPI();
 }
 
-//top-menu: Tools | Data Science | Principal Component Analysis...
+//top-menu: ML | Dimension Reduction | PCA...
 //name: PCA
 //description: Principal component analysis (PCA).
-//input: dataframe table
-//input: column_list features {type: numerical}
-//input: int components = 2
-//input: bool center = true
-//input: bool scale = true
+//input: dataframe table {category: Data}
+//input: column_list features {type: numerical; category: Data}
+//input: int components = 2 {caption: Components; category: Hyperparameters} [Number of components.]
+//input: bool center = false {category: Hyperparameters} [Indicating whether the variables should be shifted to be zero centered.]
+//input: bool scale = false {category: Hyperparameters} [Indicating whether the variables should be scaled to have unit variance.]
 //output: dataframe result {action:join(table)}
 export async function PCA(table: DG.DataFrame, features: DG.ColumnList, components: number,
   center: boolean, scale: boolean): Promise<DG.DataFrame> 
 {
-  return renamePCAcolumns(await computePCA(table, features, components, center, scale));
+  const pcaTable = await computePCA(table, features, components, center, scale);
+  addPrefixToEachColumnName('PCA', pcaTable.columns);
+  return pcaTable;
+}
+
+//top-menu: ML | Dimension Reduction | UMAP...
+//name: UMAP
+//description: Uniform Manifold Approximation and Projection (UMAP).
+//input: dataframe table {category: Data}
+//input: column_list features {type: numerical; category: Data}
+//input: int components = 2 {caption: Components; category: Hyperparameters} [The number of components (dimensions) to project the data to.]
+//input: int epochs = 100 {caption: Epochs; category: Hyperparameters} [The number of epochs to optimize embeddings.]
+//input: int neighbors = 15 {caption: Neighbors; category: Hyperparameters} [The number of nearest neighbors to construct the fuzzy manifold.]
+//input: double minDist = 0.1 {caption: Minimum distance; category: Hyperparameters} [The effective minimum distance between embedded points.]
+//input: double spread = 1.0 {caption: Spread; category: Hyperparameters} [The effective scale of embedded points.]
+//output: dataframe result {action:join(table)}
+export async function UMAP(table: DG.DataFrame, features: DG.ColumnList, components: number,
+  epochs: number, neighbors: number, minDist: number, spread: number): Promise<DG.DataFrame> 
+{
+  return await computeUMAP(features, components, epochs, neighbors, minDist, spread);  
+}
+
+//top-menu: ML | Dimension Reduction | t-SNE...
+//name: t-SNE
+//description: t-distributed stochastic neighbor embedding (t-SNE).
+//input: dataframe table {category: Data}
+//input: column_list features {type: numerical; category: Data}
+//input: int components = 2 {caption: Components; category: Hyperparameters} [Dimension of the embedded space.]
+//input: double learningRate = 10 {caption: Learning rate; category: Hyperparameters} [Optimization tuning parameter. Should be in the range 10...1000.]
+//input: int perplexity = 30 {caption: Perplexity; category: Hyperparameters} [The number of nearest neighbors. Should be less than the number of samples.]
+//input: int iterations = 500 {caption: Iterations; category: Hyperparameters} [Maximum number of iterations for the optimization. Should be at least 250.]
+//output: dataframe result {action:join(table)}
+export async function tSNE(table: DG.DataFrame, features: DG.ColumnList, components: number,
+  learningRate: number, perplexity: number, iterations: number): Promise<DG.DataFrame> 
+{
+  return await computeTSNE(features, components, learningRate, perplexity, iterations);
+}
+
+//top-menu: ML | Dimension Reduction | SPE...
+//name: SPE
+//description: Stochastic proximity embedding (SPE).
+//input: dataframe table {category: Data}
+//input: column_list features {type: numerical; category: Data}
+//input: int dimension = 2 {caption: Dimension; category: Hyperparameters} [Dimension of the embedded space.]
+//input: int steps = 0 {caption: Steps; category: Hyperparameters} [Number of random selections of point pairs and distance computations between them.]
+//input: int cycles = 1000000 {caption: Cycles; category: Hyperparameters} [Number of the method cycles.]
+//input: double cutoff = 0.0 {caption: Cutoff; category: Hyperparameters} [Cutoff distance between points.]
+//input: double lambda = 2.0 {caption: Learning rate; category: Hyperparameters} [Optimization tuning parameter.]
+//output: dataframe result {action:join(table)}
+export async function SPE(table: DG.DataFrame, features: DG.ColumnList, dimension: number,
+  steps: number, cycles: number, cutoff: number, lambda: number): Promise<DG.DataFrame> 
+{
+  return await computeSPE(features, dimension, steps, cycles, cutoff, lambda);
 }
 
 //top-menu: ML | Multivariate Analysis (PLS)...

package/src/utils.ts

@@ -20,15 +20,117 @@ const INCORERRECT_FEATURES_MES = 'features must be positive.';
 const INCORERRECT_SAMPLES_MES = 'samples must be positive.';
 const INCORERRECT_PERCENTAGE_MES = 'violators percentage must be from the range from 0 to 100.';
 const DATAFRAME_IS_TOO_BIG_MES = 'dataframe is too big.';
+const UNSUPPORTED_COLUMN_TYPE_MES = 'unsupported column type: ';
+const INCORRECT_MIN_DIST_MES = 'min distance must be positive.';
+const INCORRECT_SPREAD_MES = 'spread must be positive.';
+const INCORRECT_EPOCH_MES = 'number of epoch must be at least 1.';
+const INCORRECT_NEIBORS_MES = 'number of neibors must be at least 2 and not greater than samples count.';
+const INCORRECT_ITERATIONS_MES = 'number of iterations must be at least 1.';
+const INCORRECT_LEARNING_RATE_MES = 'learning rate must be positive.';
+const INCORRECT_PERPLEXITY_MES = 'perplexity must be at least 2 and not greater than samples count.';
+const INCORRECT_STEPS_MES = 'steps must be non-negative.';
+const INCORRECT_CYCLES_MES = 'cycles must be positive.';
+const INCORRECT_CUTOFF_MES = 'cutoff must be non-negative.'
 
-// Check components count (PCA, PLS)
-export function checkComponenets(features: DG.ColumnList, components: number): void {
+// Check column type
+export function checkColumnType(col: DG.Column): void {
+  if ((col.type != DG.COLUMN_TYPE.FLOAT) && (col.type != DG.COLUMN_TYPE.INT))
+    throw new Error(UNSUPPORTED_COLUMN_TYPE_MES + col.type);
+}
+
+// Check dimension reducer inputs
+export function checkDimensionReducerInputs(features: DG.ColumnList, components: number): void {
   if (components < COMP_MIN)
     throw new Error(COMP_POSITVE_MES);
 
   if (components > features.length)
     throw new Error(COMP_EXCESS);
 
+  for (const col of features)
+    checkColumnType(col);
+}
+
+// Check UMAP inputs
+export function checkUMAPinputs(features: DG.ColumnList, components: number, epochs: number,
+  neighbors: number, minDist: number, spread: number): void 
+{
+  // General dim reducer checks
+  checkDimensionReducerInputs(features, components);
+
+  // Check data total size
+  if (features.length * features.byIndex(0).length > MAX_ELEMENTS_COUNT)
+    throw new Error(DATAFRAME_IS_TOO_BIG_MES);
+
+  // UMAP specific checks
+
+  if (minDist <= 0)
+    throw new Error(INCORRECT_MIN_DIST_MES);
+
+  if (spread <= 0)
+    throw new Error(INCORRECT_SPREAD_MES);
+
+  if (epochs < 1)
+    throw new Error(INCORRECT_EPOCH_MES);
+
+  if ((neighbors < 2) || (neighbors > features.byIndex(0).length))
+    throw new Error(INCORRECT_NEIBORS_MES);  
+}
+
+// Check t-SNE inputs
+export function checkTSNEinputs(features: DG.ColumnList, components: number, 
+  learningRate: number, perplexity: number, iterations: number): void 
+{
+  // General dim reducer checks
+  checkDimensionReducerInputs(features, components);
+
+  // Check data total size
+  if (features.length * features.byIndex(0).length > MAX_ELEMENTS_COUNT)
+    throw new Error(DATAFRAME_IS_TOO_BIG_MES);
+
+  // t-SNE specific checks
+
+  if (learningRate < 0)
+    throw new Error(INCORRECT_LEARNING_RATE_MES);
+
+  if (iterations < 1)
+    throw new Error(INCORRECT_ITERATIONS_MES);
+
+  if ((perplexity < 2) || (perplexity > features.byIndex(0).length))
+    throw new Error(INCORRECT_PERPLEXITY_MES);  
+}
+
+// Check SPE inputs
+export function checkSPEinputs(features: DG.ColumnList, dimension: number,
+  steps: number, cycles: number, cutoff: number, lambda: number): void 
+{
+  // General dim reducer checks
+  checkDimensionReducerInputs(features, dimension);
+
+  // Check data total size
+  if (features.length * features.byIndex(0).length > MAX_ELEMENTS_COUNT)
+    throw new Error(DATAFRAME_IS_TOO_BIG_MES);
+
+  // SPE specific checks
+
+  if (steps < 0)
+    throw new Error(INCORRECT_STEPS_MES);
+
+  if (cycles <= 0)
+    throw new Error(INCORRECT_CYCLES_MES);
+
+  if (cutoff < 0)
+    throw new Error(INCORRECT_CUTOFF_MES);
+
+  if (lambda <= 0)
+    throw new Error(INCORRECT_LEARNING_RATE_MES);
+}
+
+// Check wasm dimension reducer inputs
+export function checkWasmDimensionReducerInputs(features: DG.ColumnList, components: number): void {
+  // General dim reducer checks
+  checkDimensionReducerInputs(features, components);
+
+  // Check data total size
   if (features.length * features.byIndex(0).length > MAX_ELEMENTS_COUNT)
     throw new Error(DATAFRAME_IS_TOO_BIG_MES);
 }
@@ -49,3 +151,28 @@ export function checkGeneratorSVMinputs(samplesCount: number, featuresCount: num
   if ((violatorsPercentage < PERCENTAGE_MIN) || (violatorsPercentage > PERCENTAGE_MAX))
     throw new Error(INCORERRECT_PERCENTAGE_MES);
 }
+
+// Returns rows of column data
+export function getRowsOfNumericalColumnns(columnList: DG.ColumnList): any[][] {
+  const columns = columnList.toList();
+  const rowCount = columns[0].length;
+  const colCount = columns.length;  
+
+  const output = [] as any[][];
+
+  for (let i = 0; i < rowCount; ++i)
+    output.push(Array(colCount));
+
+  for (let j = 0; j < colCount; ++j) {
+    const col = columns[j];
+
+    checkColumnType(col);
+
+    const array = col.getRawData();
+
+    for (let i = 0; i < rowCount; ++i)
+      output[i][j] = array[i];
+  }
+
+  return output;
+}

package/src/workers/tsne-worker.ts

@@ -0,0 +1,20 @@
+// Worker for the method t-SNE
+
+import {TSNE} from '@keckelt/tsne';
+
+onmessage = async function (evt) {
+  const tsne = new TSNE({
+    epsilon: evt.data.options.learningRate,
+    perplexity: evt.data.options.perplexity,
+    dim: evt.data.options.components
+  });
+      
+  tsne.initDataRaw(evt.data.data);
+
+  const iterCount = evt.data.options.iterations;
+      
+  for(let i = 0; i < iterCount; ++i)
+    tsne.step();
+
+  postMessage({'embeddings': tsne.getSolution()});
+}

package/src/workers/umap-worker.ts

@@ -0,0 +1,9 @@
+// Worker for the method UMAP
+
+import { UMAP } from 'umap-js';
+
+onmessage = async function (evt) {
+  const umap = new UMAP(evt.data.options);
+  const embeddings = umap.fit(evt.data.data);
+  postMessage({'embeddings': embeddings});
+}

package/src/EDAtools.ts

@@ -1,46 +0,0 @@
-// Exploratory data analysis (EDA) tools
-
-import * as grok from 'datagrok-api/grok';
-import * as ui from 'datagrok-api/ui';
-import * as DG from 'datagrok-api/dg';
-
-import {_principalComponentAnalysisInWebWorker,
-  _partialLeastSquareRegressionInWebWorker} from '../wasm/EDAAPI';
-
-import {checkComponenets, checkGeneratorSVMinputs} from './utils';
-
-// Principal components analysis (PCA)
-export async function computePCA(table: DG.DataFrame, features: DG.ColumnList, components: number,
-  center: boolean, scale: boolean): Promise<DG.DataFrame> 
-{
-  checkComponenets(features, components);
-
-  const centerNum = center ? 1 : 0;
-  const scaleNum = scale ? 1 : 0;
-
-  let _output: any;
-  let _promise = _principalComponentAnalysisInWebWorker(table, features, components, centerNum, scaleNum);
-
-  await _promise.then(
-    _result => { _output = _result; },
-    _error => {  throw new Error (`Error: ${_error}`); }
-  );
-
-  return _output;  
-} 
-
-// Partial least square regression (PLS)
-export async function computePLS(table: DG.DataFrame, features: DG.ColumnList, predict: DG.Column, components: number): Promise<any> 
-{
-  checkComponenets(features, components);
-
-  let _output: any;
-  let _promise = _partialLeastSquareRegressionInWebWorker(table, features, predict, components);
-
-  await _promise.then(
-    _result => { _output = _result; },    
-    _error => {  throw new Error (`Error: ${_error}`); }
-  );
-
-  return _output;
-}