@datagrok/eda 1.1.28 → 1.1.29

package/src/softmax-classifier.ts

@@ -0,0 +1,412 @@
+// Softmax classifier (multinomial logistic regression): https://en.wikipedia.org/wiki/Multinomial_logistic_regression
+
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+
+import {_fitSoftmax} from '../wasm/EDAAPI';
+
+const ROWS_EXTRA = 1;
+const COLS_EXTRA = 2;
+const MIN_COLS_COUNT = 1 + COLS_EXTRA;
+const AVGS_NAME = 'Avg-s';
+const STDEVS_NAME = 'Stddev-s';
+const PRED_NAME = 'predicted';
+const DEFAULT_LEARNING_RATE = 1;
+const DEFAULT_ITER_COUNT = 100;
+const DEFAULT_PENALTY = 0.1;
+const DEFAULT_TOLERANCE = 0.001;
+const BYTES_PER_MODEL_SIZE = 4;
+
+/** Train data sizes */
+type DataSpecification = {
+  classesCount: number,
+  featuresCount: number,
+};
+
+/** Target labels specification */
+type TargetLabelsData = {
+  oneHot: Array<Uint8Array>,
+  weights: Uint32Array,
+};
+
+/** Interactivity tresholds */
+enum INTERACTIVITY {
+  MAX_SAMLPES = 50000,
+  MAX_FEATURES = 100,
+};
+
+/** Softmax classifier */
+export class SoftmaxClassifier {
+  /** Check applicability */
+  static isApplicable(features: DG.ColumnList, predictColumn: DG.Column): boolean {
+    for (const col of features) {
+      if (!col.matches('numerical'))
+        return false;
+    }
+
+    return (predictColumn.type === DG.COLUMN_TYPE.STRING);
+  }
+
+  /** Check interactivity */
+  static isInteractive(features: DG.ColumnList, predictColumn: DG.Column): boolean {
+    return (features.length <= INTERACTIVITY.MAX_FEATURES) &&
+      (predictColumn.length <= INTERACTIVITY.MAX_SAMLPES);
+  }
+
+  private avgs: Float32Array;
+  private stdevs: Float32Array;
+  private categories: string[];
+  private params: Float32Array[] | undefined = undefined;
+  private classesCount = 1;
+  private featuresCount = 1;
+
+  constructor(specification?: DataSpecification, packedModel?: Uint8Array) {
+    if (specification !== undefined) { // Create empty model
+      /** features count */
+      const n = specification.featuresCount;
+
+      /** classes count */
+      const c = specification.classesCount;
+
+      if (n < 1)
+        throw new Error('Incorrect features count');
+
+      if (c < 1)
+        throw new Error('Incorrect classes count');
+
+      /** length of arrays */
+      const len = n + ROWS_EXTRA;
+
+      // Init model routine
+      this.avgs = new Float32Array(len);
+      this.stdevs = new Float32Array(len);
+      this.categories = new Array<string>(len);
+      this.featuresCount = n;
+      this.classesCount = c;
+    } else if (packedModel !== undefined) { // Get classifier from packed model (bytes)
+      try {
+        // Extract model's bytes count
+        const sizeArr = new Uint32Array(packedModel.buffer, 0, 1);
+        const bytesCount = sizeArr[0];
+
+        // Model's bytes
+        const modelBytes = new Uint8Array(packedModel.buffer, BYTES_PER_MODEL_SIZE, bytesCount);
+
+        const modelDf = DG.DataFrame.fromByteArray(modelBytes);
+        const columns = modelDf.columns;
+        const colsCount = columns.length;
+
+        if (colsCount < MIN_COLS_COUNT)
+          throw new Error('incorrect columns count');
+
+        this.classesCount = colsCount - COLS_EXTRA;
+        this.featuresCount = modelDf.rowCount - ROWS_EXTRA;
+
+        const c = this.classesCount;
+
+        // extract params & categories
+        this.params = new Array<Float32Array>(c);
+        this.categories = new Array<string>(modelDf.rowCount);
+
+        for (let i = 0; i < c; ++i) {
+          const col = columns.byIndex(i);
+          this.categories[i] = col.name;
+
+          if (col.type !== DG.COLUMN_TYPE.FLOAT)
+            throw new Error(`Incorrect input column type. Expected: float, passed: ${col.type}`);
+
+          this.params[i] = col.getRawData() as Float32Array;
+        }
+
+        // extract averages
+        const avgsCol = columns.byName(AVGS_NAME);
+        if (avgsCol.type !== DG.COLUMN_TYPE.FLOAT)
+          throw new Error('incorrect average values column type');
+        this.avgs = avgsCol.getRawData() as Float32Array;
+
+        // extract stdevs
+        const stdevsCol = columns.byName(STDEVS_NAME);
+        if (stdevsCol.type !== DG.COLUMN_TYPE.FLOAT)
+          throw new Error('incorrect standard deviations column type');
+        this.stdevs = stdevsCol.getRawData() as Float32Array;
+      } catch (e) {
+        throw new Error(`Failed to load model: ${(e instanceof Error ? e.message : 'the platform issue')}`);
+      }
+    } else
+      throw new Error('Softmax classifier not initialized');
+  }; // constructor
+
+  /** Return packed softmax classifier */
+  public toBytes(): Uint8Array {
+    if (this.params === undefined)
+      throw new Error('Non-trained model');
+
+    const c = this.classesCount;
+    const columns = new Array<DG.Column>(c + COLS_EXTRA);
+
+    // params columns
+    for (let i = 0; i < c; ++i)
+      columns[i] = DG.Column.fromFloat32Array(this.categories[i], this.params[i]);
+
+    // averages
+    columns[c] = DG.Column.fromFloat32Array(AVGS_NAME, this.avgs);
+
+    // stdevs
+    columns[c + 1] = DG.Column.fromFloat32Array(STDEVS_NAME, this.stdevs);
+
+    const modelDf = DG.DataFrame.fromColumns(columns);
+
+    const modelBytes = modelDf.toByteArray();
+    const bytesCount = modelBytes.length;
+
+    // Packed model bytes, including bytes count
+    const packedModel = new Uint8Array(bytesCount + BYTES_PER_MODEL_SIZE);
+
+    // 4 bytes for storing model's bytes count
+    const sizeArr = new Uint32Array(packedModel.buffer, 0, 1);
+    sizeArr[0] = bytesCount;
+
+    // Store model's bytes
+    packedModel.set(modelBytes, BYTES_PER_MODEL_SIZE);
+
+    return packedModel;
+  } // toBytes
+
+  /** Train classifier */
+  public async fit(features: DG.ColumnList, target: DG.Column, rate: number = DEFAULT_LEARNING_RATE,
+    iterations: number = DEFAULT_ITER_COUNT, penalty: number = DEFAULT_PENALTY, tolerance: number = DEFAULT_TOLERANCE) {
+    if (features.length !== this.featuresCount)
+      throw new Error('Training failes - incorrect features count');
+
+    if ((rate <= 0) || (iterations < 1) || (penalty <= 0) || (tolerance <= 0))
+      throw new Error('Training failes - incorrect fitting hyperparameters');
+
+    // Extract statistics & categories
+    this.extractStats(features);
+    const rowsCount = target.length;
+    const classesCount = target.categories.length;
+    const cats = target.categories;
+    for (let i = 0; i < classesCount; ++i)
+      this.categories[i] = cats[i];
+
+    try {
+      // call wasm-computations
+      const paramCols = _fitSoftmax(
+        features,
+        DG.Column.fromFloat32Array('avgs', this.avgs, this.featuresCount),
+        DG.Column.fromFloat32Array('stdevs', this.stdevs, this.featuresCount),
+        DG.Column.fromInt32Array('targets', target.getRawData() as Int32Array, rowsCount),
+        classesCount,
+        iterations, rate, penalty, tolerance,
+        this.featuresCount + 1, classesCount,
+      ).columns as DG.ColumnList;
+
+      this.params = new Array<Float32Array>(classesCount);
+      for (let i = 0; i < classesCount; ++i)
+        this.params[i] = paramCols.byIndex(i).getRawData() as Float32Array;
+    } catch (error) {
+      try { // call fitting TS-computations (if wasm failed)
+        this.params = await this.fitSoftmaxParams(
+          features,
+          target,
+          iterations,
+          rate,
+          penalty,
+          tolerance,
+        ) as Float32Array[];
+      } catch (error) {
+        throw new Error('Training failes');
+      }
+    }
+
+    if (this.params === undefined)
+      throw new Error('Training failes');
+  }; // fit
+
+  /** Extract features' stats */
+  private extractStats(features: DG.ColumnList): void {
+    let j = 0;
+
+    for (const col of features) {
+      if ((col.type !== DG.COLUMN_TYPE.INT) && (col.type !== DG.COLUMN_TYPE.FLOAT))
+        throw new Error('Training failes - incorrect features type');
+
+      this.avgs[j] = col.stats.avg;
+      this.stdevs[j] = col.stats.stdev;
+
+      ++j;
+    }
+  } // extractStats
+
+  /** Retrun normalized features */
+  private normalized(features: DG.ColumnList): Array<Float32Array> {
+    const m = features.byIndex(0).length;
+
+    const X = new Array<Float32Array>(m);
+
+    for (let i = 0; i < m; ++i)
+      X[i] = new Float32Array(this.featuresCount);
+
+    let j = 0;
+    for (const col of features) {
+      if ((col.type !== DG.COLUMN_TYPE.INT) && (col.type !== DG.COLUMN_TYPE.FLOAT))
+        throw new Error('Training failes - incorrect features type');
+
+      const raw = col.getRawData();
+      const avg = this.avgs[j];
+      const stdev = this.stdevs[j];
+
+      if (stdev > 0) {
+        for (let i = 0; i < m; ++i)
+          X[i][j] = (raw[i] - avg) / stdev;
+      } else {
+        for (let i = 0; i < m; ++i)
+          X[i][j] = 0;
+      }
+
+      ++j;
+    }
+
+    return X;
+  } // normalized
+
+  /** Retrun normalized & transposed features */
+  private transposed(features: DG.ColumnList): Array<Float32Array> {
+    const m = features.byIndex(0).length;
+    const n = this.featuresCount;
+
+    const X = new Array<Float32Array>(n);
+
+    for (let i = 0; i < n; ++i)
+      X[i] = new Float32Array(m);
+
+    let j = 0;
+    for (const col of features) {
+      if ((col.type !== DG.COLUMN_TYPE.INT) && (col.type !== DG.COLUMN_TYPE.FLOAT))
+        throw new Error('Training failes - incorrect features type');
+
+      const raw = col.getRawData();
+      const avg = this.avgs[j];
+      const stdev = this.stdevs[j];
+
+      if (stdev > 0) {
+        for (let i = 0; i < m; ++i)
+          X[j][i] = (raw[i] - avg) / stdev;
+      } else {
+        for (let i = 0; i < m; ++i)
+          X[j][i] = 0;
+      }
+
+      ++j;
+    }
+
+    return X;
+  } // transposed
+
+  /** Return one-hot vectors and classes weights */
+  private preprocessedTargets(target: DG.Column): TargetLabelsData {
+    if (target.type !== DG.COLUMN_TYPE.STRING)
+      throw new Error('Training failes - incorrect target type');
+
+    const c = this.classesCount;
+    const m = target.length;
+    const raw = target.getRawData();
+
+    const Y = new Array<Uint8Array>(m);
+    const weights = new Uint32Array(c).fill(0);
+
+    for (let i = 0; i < m; ++i)
+      Y[i] = new Uint8Array(c).fill(0);
+
+    for (let i = 0; i < m; ++i) {
+      Y[i][raw[i]] = 1;
+      ++weights[raw[i]];
+    }
+
+    return {
+      oneHot: Y,
+      weights: weights,
+    };
+  } // getOneHot
+
+  /** Return prediction column */
+  public predict(features: DG.ColumnList): DG.Column {
+    if (this.params === undefined)
+      throw new Error('Non-trained model');
+
+    if (features.length !== this.featuresCount)
+      throw new Error('Predcition fails: incorrect features count');
+
+    // Normalize features
+    const X = this.normalized(features);
+
+    // Routine items
+    const m = X.length;
+    const n = this.featuresCount;
+    const c = this.classesCount;
+    let xBuf: Float32Array;
+    let wBuf: Float32Array;
+    const Z = new Float32Array(c);
+    let sum: number;
+    let max: number;
+    let argMax: number;
+    const predClass = new Array<string>(m);
+
+    // get prediction for each sample
+    for (let j = 0; j < m; ++j) {
+      xBuf = X[j];
+      sum = 0;
+
+      for (let i = 0; i < c; ++i) {
+        wBuf = this.params[i];
+        sum = wBuf[n];
+
+        for (let k = 0; k < n; ++k)
+          sum += wBuf[k] * xBuf[k];
+
+        Z[i] = Math.exp(sum);
+      }
+
+      max = Z[0];
+      argMax = 0;
+
+      for (let k = 1; k < c; ++k) {
+        if (max < Z[k]) {
+          max = Z[k];
+          argMax = k;
+        }
+      }
+
+      predClass[j] = this.categories[argMax];
+    }
+
+    return DG.Column.fromStrings(PRED_NAME, predClass);
+  }
+
+  /** Fit params in the webworker */
+  private async fitSoftmaxParams(features: DG.ColumnList, target: DG.Column,
+    iterations: number, rate: number, penalty: number, tolerance: number) {
+    const targetData = this.preprocessedTargets(target);
+
+    return new Promise((resolve, reject) => {
+      const worker = new Worker(new URL('./workers/softmax-worker.ts', import.meta.url));
+      worker.postMessage({
+        features: this.normalized(features),
+        transposed: this.transposed(features),
+        oneHot: targetData.oneHot,
+        classesWeights: targetData.weights,
+        targetRaw: target.getRawData(),
+        iterations: iterations,
+        rate: rate,
+        penalty: penalty,
+        tolerance: tolerance,
+      });
+      worker.onmessage = function(e) {
+        worker.terminate();
+        resolve(e.data.params);
+        console.log(`Loss: ${e.data.loss}`);
+      };
+    });
+  }
+}; // SoftmaxClassifier

package/src/svm.ts

@@ -75,19 +75,11 @@ const FEATURES_COUNT_NAME = 'Features count';
 const TRAIN_SAMPLES_COUNT_NAME = 'Train samples count';
 const TRAIN_ERROR = 'Train error, %';
 const KERNEL_TYPE_TO_NAME_MAP = ['linear', 'polynomial', 'RBF', 'sigmoid'];
-const POSITIVE_NAME = 'positive (P)';
-const NEGATIVE_NAME = 'negative (N)';
-const PREDICTED_POSITIVE_NAME = 'predicted positive (PP)';
-const PREDICTED_NEGATIVE_NAME = 'predicted negative (PN)';
 const SENSITIVITY = 'Sensitivity';
 const SPECIFICITY = 'Specificity';
 const BALANCED_ACCURACY = 'Balanced accuracy';
 const POSITIVE_PREDICTIVE_VALUE = 'Positive predicitve value';
 const NEGATIVE_PREDICTIVE_VALUE = 'Negative predicitve value';
-const ML_REPORT = 'Model report';
-const ML_REPORT_PREDICTED_LABELS = 'Predicted labels';
-const ML_REPORT_TRAIN_LABELS = 'Train labels';
-const ML_REPORT_CORRECTNESS = 'Prediction correctness';
 const PREDICTION = 'prediction';
 
 // Pack/unpack constants
@@ -277,11 +269,11 @@ export async function getTrainedModel(hyperparameters: any, df: DG.DataFrame, la
 
   if (labels.categories.length != 2)
     throw new Error(WRONG_LABELS_MESSAGE);
-  let labelNumeric : DG.Column = DG.Column.float(labels.name, labels.length);
-  for (var i = 0; i < labels.length; i++)
+  const labelNumeric : DG.Column = DG.Column.float(labels.name, labels.length);
+  for (let i = 0; i < labels.length; i++)
     labelNumeric.set(i, labels.get(i) == labels.categories[0] ? -1.0 : 1.0, false);
 
-  let model = await trainAndAnalyzeModel(hyperparameters, columns, labelNumeric);
+  const model = await trainAndAnalyzeModel(hyperparameters, columns, labelNumeric);
   model.realLabels = labels;
   return model;
 }
@@ -306,19 +298,6 @@ function getModelInfo(model: any): DG.DataFrame {
   ]);
 }
 
-// Get dataframe with confusion matrix
-function getConfusionMatrixDF(model: any): DG.DataFrame {
-  const data = model.confusionMatrix.getRawData();
-
-  return DG.DataFrame.fromColumns([
-    DG.Column.fromStrings('', [POSITIVE_NAME, NEGATIVE_NAME]),
-    DG.Column.fromList('int', PREDICTED_POSITIVE_NAME,
-      [data[TRUE_POSITIVE_INDEX], data[FALSE_POSITIVE_INDEX]]),
-    DG.Column.fromList('int', PREDICTED_NEGATIVE_NAME,
-      [data[FALSE_NEGATIVE_INDEX], data[TRUE_NEGATIVE_INDEX]]),
-  ]);
-}
-
 // Show training report
 export function showTrainReport(df: DG.DataFrame, packedModel: any): HTMLElement {
   const model = getUnpackedModel(packedModel);
@@ -335,7 +314,7 @@ export function getPackedModel(model: any): any {
   const realLabelsSize = BYTES + realLabelsBuffer.length + 4 - realLabelsBuffer.length % 4;
   const modelInfoBuffer = getModelInfo(model).toByteArray();
   const modelInfoSize = BYTES + modelInfoBuffer.length + 4 - modelInfoBuffer.length % 4;
-  
+
   /*let bufferSize = BYTES * (7 + featuresCount * samplesCount
     + 3 * featuresCount + 2 * samplesCount);*/
 
@@ -472,7 +451,7 @@ function getUnpackedModel(packedModel: any): any {
   offset += BYTES;
   const modelInfo = DG.DataFrame.fromByteArray(new Uint8Array(modelBytes, offset, modelInfoSize));
   offset += modelInfoBytesSize;
-  
+
   const model = {kernelType: header[MODEL_KERNEL_INDEX],
     kernelParams: kernelParams,
     trainLabels: trainLabels,
@@ -482,7 +461,7 @@ function getUnpackedModel(packedModel: any): any {
     modelParams: modelParams,
     modelWeights: modelWeights,
     normalizedTrainData: normalizedTrainData,
-    modelInfo: modelInfo
+    modelInfo: modelInfo,
   };
 
   return model;
@@ -491,13 +470,12 @@ function getUnpackedModel(packedModel: any): any {
 // Wrapper for combining the function "predict" with Datagrok predicitve tools
 export async function getPrediction(df: DG.DataFrame, packedModel: any): Promise<DG.DataFrame> {
   const model = getUnpackedModel(new Uint8Array(packedModel));
-
   const resNumeric = await predict(model, df.columns);
   const res = DG.Column.string(PREDICTION, resNumeric.length);
   const categories = model.realLabels.categories;
-  for (var i = 0; i < res.length; i++) {
+  for (let i = 0; i < res.length; i++)
     res.set(i, resNumeric.get(i) == -1 ? categories[0] : categories[1]);
-  }
+
 
   return DG.DataFrame.fromColumns([res]);
 } // getPrediction
@@ -507,12 +485,12 @@ export function isApplicableSVM(df: DG.DataFrame, labels: DG.Column): boolean {
   const columns = df.columns;
   if (!labels.matches('categorical') || labels.categories.length > 2)
     return false;
-  var res: boolean = true;
-  for (var i = 0; i < columns.length; i++)
+  let res: boolean = true;
+  for (let i = 0; i < columns.length; i++)
     res = res && (columns.byIndex(i).matches('numerical'));
   return res;
 }
 
 export function isInteractiveSVM(df: DG.DataFrame, labels: DG.Column): boolean {
   return df.rowCount <= 1000;
-}
+}

package/src/workers/softmax-worker.ts

@@ -0,0 +1,146 @@
+// Worker for softmax training
+
+onmessage = async function(evt) {
+  const X = evt.data.features;
+  const transposedX = evt.data.transposed;
+  const Y = evt.data.oneHot;
+  const classesWeights = evt.data.classesWeights;
+  const iterations = evt.data.iterations;
+  const rate = evt.data.rate;
+  const penalty = evt.data.penalty;
+  const tolerance = evt.data.tolerance;
+  const targetRaw = evt.data.targetRaw;
+  let loss = 0;
+  let lossPrev = 0;
+
+  const n = transposedX.length;
+  const m = X.length;
+  const c = classesWeights.length;
+
+  // 1. Init params
+  // Xavier initialization scale value
+  const xavierScale = 2 * Math.sqrt(6.0 / (c + n));
+  const params = new Array<Float32Array>(c);
+  for (let i = 0; i < c; ++i) {
+    const current = new Float32Array(n + 1);
+
+    // initialize bias, b
+    current[n] = 0;
+
+    //Xavier initialization of weights, w
+    for (let j = 0; j < n; ++j)
+      current[j] = (Math.random() - 0.5) * xavierScale;
+
+    params[i] = current;
+  }
+
+  // 2. Fitting
+
+  // Routine
+  let xBuf: Float32Array;
+  let wBuf: Float32Array;
+  let zBuf: Float32Array;
+  let sum: number;
+  let sumExp: number;
+  let yTrue: Uint8Array;
+  let yPred: Float32Array;
+  let dWbuf: Float32Array;
+  const Z = new Array<Float32Array>(m);
+  for (let i = 0; i < m; ++i)
+    Z[i] = new Float32Array(c);
+  const dZ = new Array<Float32Array>(c);
+  for (let i = 0; i < c; ++i)
+    dZ[i] = new Float32Array(m);
+  const dW = new Array<Float32Array>(c);
+  for (let i = 0; i < c; ++i)
+    dW[i] = new Float32Array(n + 1);
+
+  // Fitting
+  for (let iter = 0; iter < iterations; ++iter) {
+    // 2.1) Forward propagation
+    for (let j = 0; j < m; ++j) {
+      xBuf = X[j];
+      zBuf = Z[j];
+      sum = 0;
+      sumExp = 0;
+
+      for (let i = 0; i < c; ++i) {
+        wBuf = params[i];
+        sum = wBuf[n];
+
+        for (let k = 0; k < n; ++k)
+          sum += wBuf[k] * xBuf[k];
+
+        zBuf[i] = Math.exp(sum) * classesWeights[i];
+        sumExp += zBuf[i];
+      }
+
+      for (let i = 0; i < c; ++i)
+        zBuf[i] /= sumExp;
+    }
+
+    // 2.2) Loss
+    loss = 0;
+
+    for (let i = 0; i < m; ++i)
+      loss += -Math.log(Z[i][targetRaw[i]]);
+
+    loss /= m;
+
+    if (Math.abs(loss - lossPrev) < tolerance)
+      break;
+
+    lossPrev = loss;
+
+    // 2.3) Backward propagation
+
+    // 2.3.1) dZ
+    for (let j = 0; j < m; ++j) {
+      yPred = Z[j];
+      yTrue = Y[j];
+
+      for (let i = 0; i < c; ++i)
+        dZ[i][j] = yPred[i] - yTrue[i];
+    }
+
+    // 2.3.2) dB
+    for (let i = 0; i < c; ++i) {
+      sum = 0;
+      zBuf = dZ[i];
+
+      for (let j = 0; j < m; ++j)
+        sum += zBuf[j];
+
+      dW[i][n] = sum / m;
+    }
+
+    // 2.3.3) dW
+    for (let i = 0; i < c; ++i) {
+      zBuf = dZ[i];
+      wBuf = dW[i];
+
+      for (let j = 0; j < n; ++j) {
+        xBuf = transposedX[j];
+
+        sum = 0;
+        for (let k = 0; k < m; ++k)
+          sum += zBuf[k] * xBuf[k];
+
+        wBuf[j] = sum / m;
+      }
+    }
+
+    // 2.4) Update weights
+    for (let i = 0; i < c; ++i) {
+      wBuf = params[i];
+      dWbuf = dW[i];
+
+      for (let j = 0; j < n; ++j)
+        wBuf[j] = (1 - rate * penalty / m) * wBuf[j] - rate * dWbuf[j];
+
+      wBuf[n] -= rate * dWbuf[n];
+    }
+  } // for iter
+
+  postMessage({'params': params, 'loss': loss});
+};