@datagrok/eda 1.4.11 → 1.4.12

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@datagrok/eda",
   "friendlyName": "EDA",
-  "version": "1.4.11",
+  "version": "1.4.12",
   "description": "Exploratory Data Analysis Tools",
   "dependencies": {
     "@datagrok-libraries/math": "^1.2.6",
     "@datagrok-libraries/ml": "^6.10.8",
+    "@datagrok-libraries/statistics": "^1.10.0",
     "@datagrok-libraries/tutorials": "^1.7.4",
     "@datagrok-libraries/utils": "^4.6.5",
     "@keckelt/tsne": "^1.0.2",
@@ -14,6 +15,7 @@
     "datagrok-api": "^1.26.3",
     "dayjs": "^1.11.9",
     "jstat": "^1.9.6",
+    "mathjs": "^15.1.0",
     "source-map-loader": "^4.0.1",
     "umap-js": "^1.3.3",
     "worker-loader": "^3.0.8",

package/src/anova/anova-tools.ts

@@ -1,4 +1,4 @@
-// Analysis of Variances (ANOVA): computations
+// Analysis of Variances (ANOVA) - computations
 
 /* REFERENCES
 

package/src/anova/anova-ui.ts

@@ -1,4 +1,4 @@
-// Analysis of Variances (ANOVA): UI
+// Analysis of Variances (ANOVA) - UI
 
 import * as grok from 'datagrok-api/grok';
 import * as ui from 'datagrok-api/ui';

package/src/package-api.ts

@@ -274,4 +274,18 @@ export namespace funcs {
   export async function paretoFrontViewer(): Promise<any> {
     return await grok.functions.call('EDA:ParetoFrontViewer', {});
   }
+
+  /**
+  Train probabilistic multi-parameter optimization (pMPO) model
+  */
+  export async function trainPmpo(): Promise<void> {
+    return await grok.functions.call('EDA:TrainPmpo', {});
+  }
+
+  /**
+  Apply trained probabilistic multi-parameter optimization (pMPO) model to score samples
+  */
+  export async function applyPmpo(table: DG.DataFrame , file: DG.FileInfo ): Promise<void> {
+    return await grok.functions.call('EDA:ApplyPmpo', { table, file });
+  }
 }

package/src/package.g.ts

@@ -37,34 +37,34 @@ export async function PCA(table: DG.DataFrame, features: DG.ColumnList, componen
 }
 
 //name: DBSCAN clustering
-//tags: dim-red-postprocessing-function
 //input: column col1 
 //input: column col2 
 //input: double epsilon = 0.01 { description: Minimum distance between two points to be considered as in the same neighborhood. }
 //input: int minimumPoints = 5 { description: Minimum number of points to form a dense region. }
 //meta.defaultPostProcessingFunction: true
+//meta.role: dimRedPostprocessingFunction
 export async function dbscanPostProcessingFunction(col1: DG.Column, col2: DG.Column, epsilon: number, minimumPoints: number) : Promise<void> {
   await PackageFunctions.dbscanPostProcessingFunction(col1, col2, epsilon, minimumPoints);
 }
 
 //name: None (number)
-//tags: dim-red-preprocessing-function
 //input: column col 
 //input: string _metric { optional: true }
 //output: object result
 //meta.supportedTypes: int,float,double,qnum
 //meta.supportedDistanceFunctions: Difference
+//meta.role: dimRedPreprocessingFunction
 export function numberPreprocessingFunction(col: DG.Column, _metric: string) {
   return PackageFunctions.numberPreprocessingFunction(col, _metric);
 }
 
 //name: None (string)
-//tags: dim-red-preprocessing-function
 //input: column col 
 //input: string _metric { optional: true }
 //output: object result
 //meta.supportedTypes: string
 //meta.supportedDistanceFunctions: One-Hot,Levenshtein,Hamming
+//meta.role: dimRedPreprocessingFunction
 export function stringPreprocessingFunction(col: DG.Column, _metric: string) {
   return PackageFunctions.stringPreprocessingFunction(col, _metric);
 }
@@ -102,9 +102,9 @@ export async function MCLClustering(df: DG.DataFrame, cols: DG.Column[], metrics
 
 //name: MCL
 //description: Markov clustering viewer
-//tags: viewer
 //output: viewer result
 //meta.showInGallery: false
+//meta.role: viewer
 export function markovClusteringViewer() : any {
   return PackageFunctions.markovClusteringViewer();
 }
@@ -532,9 +532,22 @@ export function paretoFront() : void {
 
 //name: Pareto front
 //description: Pareto front viewer
-//tags: viewer
 //output: viewer result
 //meta.icon: icons/pareto-front-viewer.svg
+//meta.role: viewer
 export function paretoFrontViewer() : any {
   return PackageFunctions.paretoFrontViewer();
 }
+
+//description: Train probabilistic multi-parameter optimization (pMPO) model
+//top-menu: Chem | Calculate | Train pMPO...
+export function trainPmpo() : void {
+  PackageFunctions.trainPmpo();
+}
+
+//description: Apply trained probabilistic multi-parameter optimization (pMPO) model to score samples
+//input: dataframe table 
+//input: file file 
+export async function applyPmpo(table: DG.DataFrame, file: DG.FileInfo) : Promise<void> {
+  await PackageFunctions.applyPmpo(table, file);
+}

package/src/package.ts

@@ -38,6 +38,8 @@ import {initXgboost} from '../wasm/xgbooster';
 import {XGBooster} from './xgbooster';
 import {ParetoOptimizer} from './pareto-optimization/pareto-optimizer';
 import {ParetoFrontViewer} from './pareto-optimization/pareto-front-viewer';
+import {Pmpo} from './probabilistic-scoring/prob-scoring';
+import {loadPmpoParams} from './probabilistic-scoring/pmpo-utils';
 
 export const _package = new DG.Package();
 export * from './package.g';
@@ -113,12 +115,7 @@ export class PackageFunctions {
 
 
   @grok.decorators.func({
-    'meta': {
-      'defaultPostProcessingFunction': 'true',
-    },
-    'tags': [
-      'dim-red-postprocessing-function',
-    ],
+    'meta': {'defaultPostProcessingFunction': 'true', role: 'dimRedPostprocessingFunction'},
     'name': 'DBSCAN clustering',
   })
   static async dbscanPostProcessingFunction(
@@ -148,8 +145,8 @@ export class PackageFunctions {
     'meta': {
       'supportedTypes': 'int,float,double,qnum',
       'supportedDistanceFunctions': 'Difference',
+      'role': 'dimRedPreprocessingFunction'
     },
-    'tags': ['dim-red-preprocessing-function'],
     'name': 'None (number)',
     'outputs': [{name: 'result', type: 'object'}],
   })
@@ -166,8 +163,8 @@ export class PackageFunctions {
     'meta': {
       'supportedTypes': 'string',
       'supportedDistanceFunctions': 'One-Hot,Levenshtein,Hamming',
+      'role': 'dimRedPreprocessingFunction'
     },
-    'tags': ['dim-red-preprocessing-function'],
     'name': 'None (string)',
     'outputs': [{name: 'result', type: 'object'}],
   })
@@ -289,10 +286,7 @@ export class PackageFunctions {
 
   @grok.decorators.func({
     'outputs': [{'name': 'result', 'type': 'viewer'}],
-    'tags': [
-      'viewer',
-    ],
-    'meta': {showInGallery: 'false'},
+    'meta': {showInGallery: 'false', role: 'viewer'},
     'name': 'MCL',
     'description': 'Markov clustering viewer',
   })
@@ -984,11 +978,48 @@ export class PackageFunctions {
   @grok.decorators.func({
     'name': 'Pareto front',
     'description': 'Pareto front viewer',
-    'tags': ['viewer'],
     'outputs': [{'name': 'result', 'type': 'viewer'}],
-    'meta': {'icon': 'icons/pareto-front-viewer.svg'},
+    'meta': {'icon': 'icons/pareto-front-viewer.svg', role: 'viewer'},
   })
   static paretoFrontViewer(): DG.Viewer {
     return new ParetoFrontViewer();
   }
+
+  @grok.decorators.func({
+    'top-menu': 'Chem | Calculate | Train pMPO...',
+    'name': 'trainPmpo',
+    'description': 'Train probabilistic multi-parameter optimization (pMPO) model',
+  })
+  static trainPmpo(): void {
+    const df = grok.shell.t;
+    if (df === null) {
+      grok.shell.warning('No dataframe is opened');
+      return;
+    }
+
+    if (!Pmpo.isTableValid(df))
+      return;
+
+    const pMPO = new Pmpo(df);
+    pMPO.runTrainingApp();
+  }
+
+  @grok.decorators.func({
+    //'top-menu': 'ML | Apply pMPO...',
+    'name': 'applyPmpo',
+    'description': 'Apply trained probabilistic multi-parameter optimization (pMPO) model to score samples',
+  })
+  static async applyPmpo(
+    @grok.decorators.param({'type': 'dataframe'}) table: DG.DataFrame,
+    @grok.decorators.param({'type': 'file'}) file: DG.FileInfo,
+  ): Promise<void> {
+    try {
+      const params = await loadPmpoParams(file);
+      const predName = table.columns.getUnusedName('pMPO score');
+      const prediction = Pmpo.predict(table, params, predName);
+      table.columns.add(prediction, true);
+    } catch (err) {
+      grok.shell.warning(`Failed to apply pMPO: ${err instanceof Error ? err.message : 'the platform issue.'}`);
+    }
+  }
 }

package/src/pareto-optimization/utils.ts

@@ -6,6 +6,7 @@ import {OPT_TYPE} from './defs';
 
 export const PALETTE = [DG.Color.darkGreen, DG.Color.yellow, DG.Color.darkRed];
 
+/** Return output color palette w.r.t. the specified type of optimization */
 export function getOutputPalette(type: OPT_TYPE): number[] {
   if (type === OPT_TYPE.MIN)
     return [...PALETTE];
@@ -13,13 +14,14 @@ export function getOutputPalette(type: OPT_TYPE): number[] {
   return [...PALETTE].reverse();
 }
 
-export function getColorScaleDiv(type: OPT_TYPE): HTMLElement {
+/** Return div with color scale description */
+export function getColorScaleDiv(type: OPT_TYPE, useMinMax: boolean = true): HTMLElement {
   const scale = ui.label('Color scale:');
   scale.style.paddingRight = '7px';
   const elems = [scale];
-  const minLbl = ui.label('min');
+  const minLbl = ui.label(useMinMax ? 'min' : 'worst');
   const midLbl = ui.label('. . .');
-  const maxLbl = ui.label('max');
+  const maxLbl = ui.label(useMinMax ? 'max' : 'best');
   const palette = getOutputPalette(type);
 
   const colorElems = [minLbl, midLbl, maxLbl].map((el, idx) => {
@@ -36,4 +38,4 @@ export function getColorScaleDiv(type: OPT_TYPE): HTMLElement {
   elems.push(...colorElems);
 
   return ui.divH(elems);
-}
+} // getColorScaleDiv

package/src/probabilistic-scoring/pmpo-defs.ts

@@ -0,0 +1,108 @@
+// Constants and type definitions for probabilistic scoring (pMPO)
+// Link: https://pmc.ncbi.nlm.nih.gov/articles/PMC4716604/
+
+/** Minimum number of samples required to compute pMPO */
+export const MIN_SAMPLES_COUNT = 10;
+
+export const PMPO_NON_APPLICABLE = 'pMPO is not applicable';
+export const PMPO_COMPUTE_FAILED = 'Failed to compute pMPO parameters';
+
+/** Basic statistics for desired and non-desired compounds */
+export type BasicStats = {
+  desAvg: number,
+  desStd: number,
+  nonDesAvg: number,
+  nonDesStd: number,
+};
+
+/** Descriptor statistics including basic stats, t-statistics and p-value */
+export type DescriptorStatistics = BasicStats & {
+  desLen: number,
+  nonSesLen: number,
+  tstat: number,
+  pValue: number,
+};
+
+/** Cutoff parameters for the basic functions of the pMPO model */
+export type Cutoff = {
+  cutoff: number,
+  cutoffDesired: number,
+  cutoffNotDesired: number,
+};
+
+/** Generalized Sigmoid parameters for the desirability functions of the pMPO model */
+export type SigmoidParams = {
+  pX0: number,
+  b: number,
+  c: number,
+};
+
+/** pMPO parameters including basic stats, cutoffs, sigmoid params, z-score, weight, intersections */
+export type PmpoParams = BasicStats & Cutoff & SigmoidParams & {
+  zScore: number,
+  weight: number,
+  intersections: number[],
+  x0: number,
+  xBound: number,
+};
+
+export type CorrelationTriple = [string, string, number];
+
+const DESIRED = 'desired';
+const NON_DESIRED = 'non-desired';
+const MEAN = 'Mean';
+const STD = 'Std';
+const T_STAT = 't-statistics';
+export const P_VAL = 'p-value';
+const MEAN_DES = `${MEAN}(${DESIRED})`;
+const MEAN_NON_DES = `${MEAN}(${NON_DESIRED})`;
+const STD_DES = `${STD}(${DESIRED})`;
+const STD_NON_DES = `${STD}(${NON_DESIRED})`;
+
+/** Map of statistic field names to their display titles  */
+export const STAT_TO_TITLE_MAP = new Map([
+  ['desAvg', MEAN_DES],
+  ['desStd', STD_DES],
+  ['nonDesAvg', MEAN_NON_DES],
+  ['nonDesStd', STD_NON_DES],
+  ['tstat', T_STAT],
+  ['pValue', P_VAL],
+]);
+
+export const DESCR_TITLE = 'Descriptor';
+export const DESCR_TABLE_TITLE = DESCR_TITLE + ' Statistics';
+export const SELECTED_TITLE = 'Selected';
+export const WEIGHT_TITLE = 'Weight';
+export const SCORES_TITLE = 'pMPO score';
+export const DESIRABILITY_COL_NAME = 'Desirability';
+
+/** Minimum p-value threshold for filtering descriptors */
+export const P_VAL_TRES_MIN = 0.01;
+
+/** Minimum R-squared threshold for filtering correlated descriptors */
+export const R2_MIN = 0.01;
+
+/** Minimum q-cutoff for descriptors in the pMPO model */
+export const Q_CUTOFF_MIN = 0.01;
+
+/** Colors used for selected and skipped descriptors */
+export enum COLORS {
+  SELECTED = 'rgb(26, 146, 26)',
+  SKIPPED = 'rgb(208, 57, 67)',
+};
+
+export const TINY = 1e-8;
+
+/** Folder path for storing pMPO models */
+export const FOLDER = 'System:AppData/Chem/mpo';
+
+/** Desirability profile properties type */
+export type DesirabilityProfileProperties = Record<string, {
+  line: [number, number][],
+  weight: number,
+  min?: number,
+  max?: number,
+}>;
+
+export const STAT_GRID_HEIGHT = 75;
+export const DESIRABILITY_COLUMN_WIDTH = 305;