@datagrok/eda 1.2.0 → 1.2.2

package/src/tests/anova-tests.ts

@@ -0,0 +1,87 @@
+// Tests for ANOVA
+
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+import {_package} from '../package-test';
+
+import {category, expect, test} from '@datagrok-libraries/utils/src/test';
+
+import {oneWayAnova, FactorizedData} from '../anova/anova-tools';
+
+const ROWS_M = 1;
+const M = 1000000;
+const TIMEOUT = 4000;
+const ALPHA = 0.05;
+const CATEGORIES = 'race';
+const FEATURES = 'height';
+const TO_VALIDATE = false;
+const ERR = 0.01;
+
+/** Validation features*/
+const FEATURES_COL = DG.Column.fromList(DG.COLUMN_TYPE.INT, 'features', [
+  9, 12, 4, 8, 7, 4, 6, 8, 2, 10, 1, 3, 4, 5, 2,
+]);
+
+/** Validation categories */
+const CATEGORIES_COL = DG.Column.fromStrings('features', [
+  'A', 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'C',
+]);
+
+/** Expected ANOVA results for the validation data */
+enum EXPECTED {
+  DF_BN = 2,
+  DF_TOT = 14,
+  DF_WN = 12,
+  SS_BN = 63.333,
+  SS_TOT = 147.333,
+  SS_WN = 84,
+  MS_BN = 31.666,
+  MS_WN = 7,
+  F_STAT = 4.523,
+  F_CRIT = 3.885,
+  P_VAL = 0.034,
+};
+
+category('ANOVA', () => {
+  test(`Performance: ${ROWS_M}M rows demog`, async () => {
+    const df = grok.data.demo.demog(ROWS_M * M);
+    const categories = df.col(CATEGORIES);
+    const features = df.col(FEATURES);
+
+    const factorized = new FactorizedData(categories!, features!, categories!.stats.uniqueCount);
+    factorized.areVarsEqual(ALPHA);
+
+    oneWayAnova(categories!, features!, ALPHA, TO_VALIDATE);
+  }, {timeout: TIMEOUT, benchmark: true});
+
+  test(`Correctness`, async () => {
+    const analysis = oneWayAnova(CATEGORIES_COL, FEATURES_COL, ALPHA, TO_VALIDATE);
+    const anova = analysis.anovaTable;
+
+    // check degrees of freedom (df-s)
+    expect(anova.dfBn, EXPECTED.DF_BN, 'Incorrect degrees of freedom: dfBn');
+    expect(anova.dfTot, EXPECTED.DF_TOT, 'Incorrect degrees of freedom: dfTot');
+    expect(anova.dfWn, EXPECTED.DF_WN, 'Incorrect degrees of freedom: dfWn');
+
+    const eq = (x: number, y: number) => Math.abs(x - y) < ERR;
+
+    // check sum of squares (ss-s)
+    expect(eq(anova.ssBn, EXPECTED.SS_BN), true, 'Incorrect sum of squares: ssBn');
+    expect(eq(anova.ssTot, EXPECTED.SS_TOT), true, 'Incorrect sum of squares: ssTot');
+    expect(eq(anova.ssWn, EXPECTED.SS_WN), true, 'Incorrect sum of squares: ssWn');
+
+    // check mean squares (ms-s)
+    expect(eq(anova.msBn, EXPECTED.MS_BN), true, 'Incorrect mean squares: msBn');
+    expect(eq(anova.msWn, EXPECTED.MS_WN), true, 'Incorrect mean squares: msWn');
+
+    // check F-statistics
+    expect(eq(anova.fStat, EXPECTED.F_STAT), true, 'Incorrect F-statistics value');
+
+    // check p-value
+    expect(eq(anova.pValue, EXPECTED.P_VAL), true, 'Incorrect p-value');
+
+    // check F-critical
+    expect(eq(analysis.fCritical, EXPECTED.F_CRIT), true, 'Incorrect F-critical');
+  }, {timeout: TIMEOUT, benchmark: true});
+});

package/src/tests/mis-vals-imputation-tests.ts

@@ -0,0 +1,58 @@
+// Tests for missing values imputation
+
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+import {_package} from '../package-test';
+
+import {category, expect, test} from '@datagrok-libraries/utils/src/test';
+
+import {MetricInfo, DISTANCE_TYPE, impute} from '../missing-values-imputation/knn-imputer';
+import {getFeatureInputSettings} from '../missing-values-imputation/ui';
+import {dataWithMissingVals} from './utils';
+
+const ROWS_K = 100;
+const K = 1000;
+const INT_COLS = 5;
+const FLOAT_COLS = 5;
+const STRING_COLS = 5;
+const MIS_VALS_COUNT = 5;
+const NEIGHBORS = 5;
+const TIMEOUT = 10000;
+const TOTAL_COLS = INT_COLS + FLOAT_COLS + STRING_COLS;
+
+const testKNN = (dist: DISTANCE_TYPE) => {
+  test(`${dist} dist, ${ROWS_K}K rows, ${TOTAL_COLS} cols, ${MIS_VALS_COUNT * TOTAL_COLS} missing vals`, async () => {
+    // Data
+    const data = dataWithMissingVals(ROWS_K * K, INT_COLS, FLOAT_COLS, STRING_COLS, MIS_VALS_COUNT);
+    const df = data.df;
+    const cols = df.columns;
+
+    // Inputs for kNN imputer
+    const targetColNames = cols.names();
+    const featuresMetrics = new Map<string, MetricInfo>();
+    const missingValsIndices = data.misValsIds;
+
+    // Imputation settings
+    for (const col of df.columns) {
+      const settings = getFeatureInputSettings(col.type as DG.COLUMN_TYPE);
+      featuresMetrics.set(col.name, {
+        weight: settings.defaultWeight,
+        type: settings.defaultMetric,
+      });
+    }
+
+    // Impute missing values & get fails
+    const failedToImput = impute(df, targetColNames, featuresMetrics, missingValsIndices, dist, NEIGHBORS, true);
+
+    // Check fails
+    let fails = 0;
+    failedToImput.forEach((inds, _) => fails += inds.length);
+    expect(fails, 0, `Failed to impute ${fails} missing values`);
+  }, {timeout: TIMEOUT, benchmark: true});
+};
+
+category(`Missing values imputation`, () => {
+  testKNN(DISTANCE_TYPE.EUCLIDEAN);
+  testKNN(DISTANCE_TYPE.MANHATTAN);
+});

package/src/tests/utils.ts

@@ -6,6 +6,10 @@ const TRESHOLD = 0.5;
 const SHIFT = 1;
 const LIMIT = 2;
 
+const MAX_INT = 10;
+const MAX_FLOAT = 10;
+const CATEGORIES = ['Alpha', 'Beta', 'Gamma', 'Delta'];
+
 /** Check lengths of columns */
 function checkLen(target: DG.Column, prediction: DG.Column): void {
   if (target.length !== prediction.length)
@@ -119,3 +123,74 @@ export function accuracy(target: DG.Column, prediction: DG.Column): number {
 
   return correctPredictions / rows;
 }
+
+/** Return dataframe with missing values */
+export function dataWithMissingVals(rows: number, intCols: number, floatCols: number,
+  strCols: number, misValCount: number): {df: DG.DataFrame, misValsIds: Map<string, number[]>} {
+  const catsCount = CATEGORIES.length;
+  const cols = [];
+  let idx = 0;
+
+  const misValsIds = new Map<string, number[]>();
+
+  for (let j = 0; j < intCols; ++j) {
+    const arr = new Int32Array(rows);
+    const name = `int #${j + 1}`;
+    const indeces: number[] = [];
+
+    for (let i = 0; i < rows; ++i)
+      arr[i] = Math.floor(Math.random() * MAX_INT);
+
+    for (let k = 0; k < misValCount; ++k) {
+      idx = Math.floor(rows * Math.random());
+      arr[idx] = DG.INT_NULL;
+      indeces.push(idx);
+    }
+
+    cols.push(DG.Column.fromInt32Array(name, arr));
+    misValsIds.set(name, indeces);
+  }
+
+  for (let j = 0; j < floatCols; ++j) {
+    const arr = new Float32Array(rows);
+    const name = `float #${j + 1}`;
+    const indeces: number[] = [];
+
+    for (let i = 0; i < rows; ++i)
+      arr[i] = Math.random() * MAX_FLOAT;
+
+    for (let k = 0; k < misValCount; ++k) {
+      idx = Math.floor(rows * Math.random());
+      arr[idx] = DG.FLOAT_NULL;
+      indeces.push(idx);
+    }
+
+    cols.push(DG.Column.fromFloat32Array(name, arr));
+    misValsIds.set(name, indeces);
+  }
+
+  for (let j = 0; j < strCols; ++j) {
+    const arr = new Array<string>(rows);
+    const name = `str #${j + 1}`;
+    const indeces: number[] = [];
+
+    for (let i = 0; i < rows; ++i)
+      arr[i] = CATEGORIES[Math.floor(Math.random() * catsCount)];
+
+    const col = DG.Column.fromStrings(name, arr);
+
+    for (let k = 0; k < misValCount; ++k) {
+      idx = Math.floor(rows * Math.random());
+      col.set(idx, null);
+      indeces.push(idx);
+    }
+
+    cols.push(col);
+    misValsIds.set(name, indeces);
+  }
+
+  return {
+    df: DG.DataFrame.fromColumns(cols),
+    misValsIds: misValsIds,
+  };
+} // tableWithMissingVals