@datagrok/eda 1.4.12 → 1.5.0

package/src/probabilistic-scoring/pmpo-utils.ts

@@ -1,17 +1,20 @@
 // Utility functions for probabilistic scoring (pMPO)
-// Link: https://pmc.ncbi.nlm.nih.gov/articles/PMC4716604/
+// Source paper https://pmc.ncbi.nlm.nih.gov/articles/PMC4716604/
 
 import * as grok from 'datagrok-api/grok';
 import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
 
+import {generateMpoFileName, MPO_PROFILE_CHANGED_EVENT} from '@datagrok-libraries/statistics/src/mpo/utils';
 import '../../css/pmpo.css';
 
 import {COLORS, DESCR_TABLE_TITLE, DESCR_TITLE, DescriptorStatistics, DesirabilityProfileProperties,
   DESIRABILITY_COL_NAME, FOLDER, P_VAL, PMPO_COMPUTE_FAILED, PmpoParams, SCORES_TITLE,
-  SELECTED_TITLE, STAT_TO_TITLE_MAP, TINY, WEIGHT_TITLE,
-  CorrelationTriple} from './pmpo-defs';
-import {computeSigmoidParamsFromX0, getCutoffs, normalPdf, sigmoidS, solveNormalIntersection} from './stat-tools';
+  SELECTED_TITLE, STAT_TO_TITLE_MAP, TINY, WEIGHT_TITLE, CorrelationTriple,
+  BASIC_RANGE_SIGMA_COEFFS, EXTENDED_RANGE_SIGMA_COEFFS, EQUALITY_SIGN,
+  PREFERABLE_CATEGORIES} from './pmpo-defs';
+import {computeSigmoidParamsFromX0, getCutoffs, gaussDesirabilityFunc, sigmoidS,
+  solveNormalIntersection} from './stat-tools';
 import {getColorScaleDiv} from '../pareto-optimization/utils';
 import {OPT_TYPE} from '../pareto-optimization/defs';
 
@@ -243,11 +246,22 @@ export function getModelParams(desired: DG.DataFrame, nonDesired: DG.DataFrame,
     // Unbiased standard deviation
     const sigmaNonDes = nonDesCol.stats.stdev * Math.sqrt((nonDesLen - 1) / nonDesLen);
 
-    // Compute cutoffs and intersections
+    // Compute cutoffs
     const cutoffs = getCutoffs(muDes, sigmaDes, muNonDes, sigmaNonDes);
+
+    // column_stats['inflection'] = np.exp(-np.square((column_stats['cutoff'] - column_stats['good_mean'])) /
+    //                                     (2 * np.square(column_stats['good_std'])))
+    // Compute inflection point
+    const inflection = Math.exp(-((cutoffs.cutoff - muDes) ** 2) / (2 * (sigmaDes ** 2)));
+
+    // Compute intersections of the two normal distributions
     const intersections = solveNormalIntersection(muDes, sigmaDes, muNonDes, sigmaNonDes);
 
-    // Compute parameters for the generalized sigmoid function
+    const b = (Math.pow(inflection, -1.0) - 1.0);
+    const n = (Math.pow(qCutoff, -1.0) - 1.0);
+    const c = Math.pow(10.0, ((Math.log10(n / b)) / (-1.0 * (muNonDes - cutoffs.cutoff))));
+
+    // Compute parameters for the generalized sigmoid function TODO: delete
 
     let x0: number | null = null;
 
@@ -279,17 +293,20 @@ export function getModelParams(desired: DG.DataFrame, nonDesired: DG.DataFrame,
       desStd: sigmaDes,
       nonDesAvg: muNonDes,
       nonDesStd: sigmaNonDes,
+      min: Math.min(desCol.stats.min, nonDesCol.stats.min),
+      max: Math.max(desCol.stats.max, nonDesCol.stats.max),
       cutoff: cutoffs.cutoff,
       cutoffDesired: cutoffs.cutoffDesired,
       cutoffNotDesired: cutoffs.cutoffNotDesired,
       pX0: sigmoidParams.pX0,
-      b: sigmoidParams.b,
-      c: sigmoidParams.c,
+      b: b,
+      c: c,
       zScore: z,
       weight: z,
       intersections: intersections,
       x0: x0,
       xBound: xBound,
+      inflection: inflection,
     });
   });
 
@@ -338,12 +355,13 @@ export async function loadPmpoParams(file: DG.FileInfo): Promise<Map<string, Pmp
  * @param name Name of the desirability profile.
  * @param description Description of the desirability profile.
  */
-export function getDesirabilityProfileJson(params: Map<string, PmpoParams>, name: string, description: string) {
+export function getDesirabilityProfileJson(params: Map<string, PmpoParams>, useSigmoidalCorrection: boolean,
+  name: string, description: string, truncatedRange: boolean): any {
   return {
     'type': 'MPO Desirability Profile',
     'name': name,
     'description': description,
-    'properties': getDesirabilityProfileProperties(params),
+    'properties': getDesirabilityProfileProperties(params, useSigmoidalCorrection, truncatedRange),
   };
 }
 
@@ -351,45 +369,23 @@ export function getDesirabilityProfileJson(params: Map<string, PmpoParams>, name
  * @param params Map of descriptor names to their pMPO parameters.
  * @param modelName Suggested model name (used as default file name).
  */
-export async function saveModel(params: Map<string, PmpoParams>, modelName: string): Promise<void> {
-  let fileName = modelName;
-  const nameInput = ui.input.string('File', {
-    value: fileName,
-    nullable: false,
-    onValueChanged: (val) => {
-      fileName = val;
-      dlg.getButton('Save').disabled = (fileName.length < 1) || (folderName.length < 1);
-    },
-  });
-
-  let folderName = FOLDER;
-  const folderInput = ui.input.string('Folder', {
-    value: folderName,
-    nullable: false,
-    onValueChanged: (val) => {
-      folderName = val;
-      dlg.getButton('Save').disabled = (fileName.length < 1) || (folderName.length < 1);
-    },
+export async function saveModel(params: Map<string, PmpoParams>, modelName: string,
+  useSigmoidalCorrection: boolean): Promise<void> {
+  const nameInput = ui.input.string('Name', {value: modelName, nullable: false});
+  const descriptionInput = ui.input.textArea('Description', {value: ' ', nullable: true});
+  const typeInput = ui.input.bool('Desirability Profile', {
+    value: true,
+    tooltipText: 'Save the model as an MPO Desirability Profile. If disabled, the model is saved in the pMPO format.',
   });
 
-  const save = async () => {
-    const path = `${folderName}/${fileName}.json`;
-    try {
-      const jsonString = JSON.stringify(objectToSave(), null, 2);
-      await grok.dapi.files.writeAsText(path, jsonString);
-      grok.shell.info(`Saved to ${path}`);
-    } catch (err) {
-      grok.shell.error(`Failed to save: ${err instanceof Error ? err.message : 'the platform issue'}.`);
-    }
-    dlg.close();
-  };
-
   const objectToSave = () => {
     if (typeInput.value) {
       return getDesirabilityProfileJson(
         params,
+        useSigmoidalCorrection,
         nameInput.value,
         descriptionInput.value,
+        false,
       );
     }
 
@@ -401,34 +397,29 @@ export async function saveModel(params: Map<string, PmpoParams>, modelName: stri
     };
   };
 
-  const modelNameInput = ui.input.string('Name', {value: modelName, nullable: true});
-  const descriptionInput = ui.input.textArea('Description', {value: ' ', nullable: true});
-  const typeInput = ui.input.bool('Desirability Profile', {
-    value: true,
-    tooltipText: 'Save the model as an MPO Desirability Profile. If disabled, the model is saved in the pMPO format.',
-  });
-
   const dlg = ui.dialog({title: 'Save model'})
-    .add(ui.h2('Path'))
-    .add(folderInput)
     .add(nameInput)
-    .add(ui.h2('Model'))
-    .add(modelNameInput)
     .add(descriptionInput)
     .add(typeInput)
     .addButton('Save', async () => {
-      const exist = await grok.dapi.files.exists(`${folderName}/${fileName}.json`);
-      if (!exist)
-        await save();
-      else {
-        // Handle overwrite confirmation
-        ui.dialog({title: 'Warning'})
-          .add(ui.label('Overwrite existing file?'))
-          .onOK(async () => await save())
-          .show();
+      try {
+        const files = await grok.dapi.files.list(FOLDER);
+        const existingFileNames = new Set(files.map((f) => f.name));
+        const fileName = generateMpoFileName(nameInput.value, existingFileNames);
+        const path = `${FOLDER}/${fileName}`;
+        const jsonString = JSON.stringify(objectToSave(), null, 2);
+        await grok.dapi.files.writeAsText(path, jsonString);
+        grok.events.fireCustomEvent(MPO_PROFILE_CHANGED_EVENT, {});
+        grok.shell.info(`Saved to ${path}`);
+      } catch (err) {
+        grok.shell.error(`Failed to save: ${err instanceof Error ? err.message : 'the platform issue'}.`);
       }
+      dlg.close();
     })
     .show();
+
+  dlg.getButton('Save').disabled = !nameInput.validate();
+  nameInput.onInput.subscribe(() => dlg.getButton('Save').disabled = !nameInput.validate());
 } // saveModel
 
 /** Adds columns with correlation coefficients between descriptors.
@@ -462,7 +453,10 @@ export function addCorrelationColumns(df: DG.DataFrame, descriptorNames: string[
   return df;
 } // addCorrelationColumns
 
-/* Sets color coding for the p-value column in the statistics table */
+/** Sets color coding for the p-value column in the statistics table
+ * @param table DataFrame with descriptor statistics.
+ * @param pValTresh P-value threshold.
+*/
 export function setPvalColumnColorCoding(table: DG.DataFrame, pValTresh: number): void {
   const pValCol = table.col(P_VAL);
   if (pValCol == null)
@@ -475,7 +469,11 @@ export function setPvalColumnColorCoding(table: DG.DataFrame, pValTresh: number)
   pValCol.meta.colors.setConditional(rules);
 } // setPvalColumnColorCoding
 
-/* Sets color coding for the p-value column in the statistics table */
+/** Sets color coding for the correlation columns in the statistics table.
+ * @param table DataFrame with descriptor statistics.
+ * @param descriptorNames List of descriptor names.
+ * @param r2Tresh R-squared threshold.
+*/
 export function setCorrColumnColorCoding(table: DG.DataFrame, descriptorNames: string[], r2Tresh: number): void {
   descriptorNames.forEach((name) => {
     const col = table.col(name);
@@ -483,8 +481,8 @@ export function setCorrColumnColorCoding(table: DG.DataFrame, descriptorNames: s
       return;
 
     const rules: Record<string, string> = {};
-    rules[`>${r2Tresh}`] = COLORS.SKIPPED;
-    rules[`=<${r2Tresh}`] = COLORS.SELECTED;
+    rules[`>=${r2Tresh}`] = COLORS.SKIPPED;
+    rules[`<${r2Tresh}`] = COLORS.SELECTED;
 
     col.meta.colors.setConditional(rules);
   });
@@ -492,20 +490,18 @@ export function setCorrColumnColorCoding(table: DG.DataFrame, descriptorNames: s
 
 /** Returns desirability profile properties for the given pMPO parameters.
  * @param params Map of descriptor names to their pMPO parameters.
+ * @param useSigmoidalCorrection Whether to use sigmoidal correction in desirability functions.
+ * @param displayProfile Whether to create a profile to be displayed in the stat grid (true - truncated range).
  */
-function getDesirabilityProfileProperties(params: Map<string, PmpoParams>) {
+function getDesirabilityProfileProperties(params: Map<string, PmpoParams>,
+  useSigmoidalCorrection: boolean, truncatedRange: boolean): DesirabilityProfileProperties {
   const props: DesirabilityProfileProperties = {};
 
-  let maxWeight = 0;
-  params.forEach((param) => maxWeight = Math.max(maxWeight, param.weight));
-
-  const scale = (maxWeight > 0) ? (1 / maxWeight) : 1;
-
   params.forEach((param, name) => {
-    const range = significantPoints(param);
+    const range = significantPoints(param, truncatedRange);
     props[name] = {
-      weight: param.weight * scale,
-      line: getLine(param),
+      weight: param.weight,
+      line: getLine(param, useSigmoidalCorrection, truncatedRange),
       min: Math.min(...range),
       max: Math.max(...range),
     };
@@ -514,67 +510,185 @@ function getDesirabilityProfileProperties(params: Map<string, PmpoParams>) {
   return props;
 } // getDesirabilityProfileProperties
 
-/** Returns array of arguments for Gaussian function centered at mu with stddev sigma. */
-function getArgsOfGaussFunc(mu: number, sigma: number): number[] {
-  return [
-    mu - 3 * sigma,
-    mu - 2.5 * sigma,
-    mu - 2 * sigma,
-    mu - 1.5 * sigma,
-    mu - sigma,
-    mu - 0.5 * sigma,
-    mu - 0.25 * sigma,
-    mu,
-    mu + 0.25 * sigma,
-    mu + 0.5 * sigma,
-    mu + sigma,
-    mu + 1.5 * sigma,
-    mu + 2 * sigma,
-    mu + 2.5 * sigma,
-    mu + 3 * sigma,
-  ];
+/** Returns array of arguments for Gaussian function centered at mu with stddev sigma.
+ * @param mu Mean of the Gaussian function.
+ * @param sigma Standard deviation of the Gaussian function.
+ * @param truncatedRange Whether to use truncated range (for interactive app) or extended range (for full profile).
+ * @return Array of arguments for the Gaussian function.
+*/
+function getArgsOfGaussFunc(mu: number, sigma: number, truncatedRange: boolean): number[] {
+  return truncatedRange ?
+    BASIC_RANGE_SIGMA_COEFFS.map((coeff) => mu + coeff * sigma) : // range for interactive app
+    EXTENDED_RANGE_SIGMA_COEFFS.map((coeff) => mu + coeff * sigma); // actual full range for desirability profile
 } // getArgsOfGaussFunc
 
-/** Returns scale factor for the given pMPO parameters and range of x values. */
-function getScale(param: PmpoParams, range: number[]): number {
-  const values = range.map((x) => basicFunction(x, param));
+/** Basic pMPO function combining Gaussian and sigmoid functions.
+ * @param x Argument.
+ * @param param pMPO parameters.
+ * @param useSigmoidalCorrection Whether to use sigmoidal correction.
+ * @return Value of the basic pMPO function at x.
+*/
+function basicFunction(x: number, param: PmpoParams, useSigmoidalCorrection: boolean): number {
+  return gaussDesirabilityFunc(x, param.desAvg, param.desStd) *
+    (useSigmoidalCorrection ? sigmoidS(x, param.cutoff, param.b, param.c) : 1);
+}
+
+/** Returns line points for the given pMPO parameters.
+ * @param param pMPO parameters.
+ * @param useSigmoidalCorrection Whether to use sigmoidal correction.
+ * @param truncatedRange Whether to use truncated range (for interactive app) or extended range (for full profile).
+ * @return Array of [x, y] points representing the desirability function line.
+*/
+function getLine(param: PmpoParams, useSigmoidalCorrection: boolean, truncatedRange: boolean): [number, number][] {
+  const range = significantPoints(param, truncatedRange);
+
+  return range.map((x) => [x, basicFunction(x, param, useSigmoidalCorrection)]);
+}
+
+/** Returns significant points for the given pMPO parameters.
+ * @param param pMPO parameters.
+ * @param truncatedRange Whether to use truncated range (for interactive app) or extended range (for full profile).
+ * @return Array of significant points for the desirability function.
+*/
+function significantPoints(param: PmpoParams, truncatedRange: boolean): number[] {
+  const points = getArgsOfGaussFunc(param.desAvg, param.desStd, truncatedRange);
+
+  /* Truncate range to show less points */
+  if (truncatedRange) {
+    const min = Math.min(param.min, param.desAvg - 3 * param.desStd);
+    const max = Math.max(param.max, param.desAvg + 3 * param.desStd);
+
+    return points
+      .filter((x) => (min <= x) && (x <= max))
+      .sort();
+  }
+
+  return points;
+} // significantPoints
 
-  return Math.max(...values);
+/** Custom error class for pMPO-related errors. */
+export class PmpoError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'PmpoError';
+  }
 }
 
-/** Basic pMPO function combining Gaussian and sigmoid functions. */
-function basicFunction(x: number, param: PmpoParams): number {
-  return normalPdf(x, param.desAvg, param.desStd) * sigmoidS(x, param.x0, param.b, param.c);
+/** Returns the initial column for the desirability input, preferring boolean columns.
+ * @param cols List of columns to choose from.
+ * @return The initial column for the desirability input.
+*/
+export function getInitCol(cols: DG.Column[]): DG.Column {
+  for (const col of cols) {
+    if ((col.type === DG.COLUMN_TYPE.BOOL) && (col.stats.stdev > 0))
+      return col;
+  }
+
+  for (const col of cols) {
+    if ((col.isNumerical) && (col.stats.stdev > 0))
+      return col;
+  }
+
+  return cols[0];
 }
 
-/** Returns line points for the given pMPO parameters. */
-function getLine(param: PmpoParams): [number, number][] {
-  //const range = getArgsOfGaussFunc(param.desAvg, param.desStd);
-  const range = significantPoints(param);
-  const scale = getScale(param, range);
+/** Returns a comparator function based on the given equality sign.
+ * @param sign Equality sign ('<', '<=', '>', '>=').
+ * @return Comparator function that takes two numbers and returns a boolean.
+*/
+function getComparator(sign: EQUALITY_SIGN): (a: number, b: number) => boolean {
+  switch (sign) {
+  case EQUALITY_SIGN.LESS: return (a, b) => a < b;
+  case EQUALITY_SIGN.LESS_OR_EQUAL: return (a, b) => a <= b;
+  case EQUALITY_SIGN.GREATER: return (a, b) => a > b;
+  case EQUALITY_SIGN.GREATER_OR_EQUAL: return (a, b) => a >= b;
+
+  default:
+    throw new Error(`Unsupported sign: ${sign}`);
+  }
+}
 
-  return range.map((x) => [x, basicFunction(x, param) / scale]);
+/** Converts a numeric column to a boolean column based on the given threshold and equality sign.
+ * @param numericCol Numeric column to convert.
+ * @param threshold Threshold value for comparison.
+ * @param sign Equality sign for comparison ('<', '<=', '>', '>=').
+ * @return Boolean column resulting from the comparison.
+*/
+export function getBoolDesirabilityColData(numericCol: DG.Column,
+  threshold: number, sign: EQUALITY_SIGN): {column: DG.Column, tooltip: string} {
+  const boolArr = new Array<boolean>(numericCol.length);
+  const numericArr = numericCol.getRawData();
+
+  const comparator = getComparator(sign);
+
+  for (let i = 0; i < numericCol.length; ++i)
+    boolArr[i] = comparator(numericArr[i], threshold);
+
+  return {
+    column: DG.Column.fromList(DG.COLUMN_TYPE.BOOL, '', boolArr),
+    tooltip: `Desirability based on the condition:\n\n **${numericCol.name} ${sign} ${threshold}**`,
+  };
 }
 
-/** Returns significant points for the given pMPO parameters. */
-function significantPoints(param: PmpoParams): number[] {
-  const start = param.desAvg - 10 * param.desStd;
-  const end = param.desAvg + 10 * param.desStd;
-  const steps = 1000;
-
-  let arg = start;
-  let func = basicFunction(arg, param);
-  let x = 0;
-  let y = 0;
-
-  for (let i = 0; i <= steps; i++) {
-    x = start + ((end - start) * i) / steps;
-    y = basicFunction(x, param);
-    if (y > func) {
-      arg = x;
-      func = y;
-    }
+/** Checks whether the desirability column is valid based on the given threshold and equality sign.
+ * @param desCol Desirability column to check.
+ * @param threshold Threshold value for comparison.
+ * @param sign Equality sign for comparison ('<', '<=', '>', '>=').
+ * @return True if the desirability column is valid, false otherwise.
+*/
+export function isDesirabilityValid(desCol: DG.Column, threshold: number, sign: EQUALITY_SIGN): boolean {
+  const min = desCol.stats.min;
+  const max = desCol.stats.max;
+
+  switch (sign) {
+  case EQUALITY_SIGN.LESS:
+    return (max >= threshold) && (min < threshold);
+
+  case EQUALITY_SIGN.LESS_OR_EQUAL:
+    return (max > threshold) && (min <= threshold);
+
+  case EQUALITY_SIGN.GREATER:
+    return (min <= threshold) && (max > threshold);
+
+  default:
+    return (min < threshold) && (max >= threshold);
   }
+}
 
-  return getArgsOfGaussFunc(arg, param.desStd);
-} // significantPoints
+/** Converts a string column to a boolean column based on the given desirable categories.
+ * @param stringCol String column to convert.
+ * @param desirableCategories List of categories that should be considered as desirable.
+ * @return Boolean column resulting from the conversion and a tooltip describing the desirability.
+*/
+export function getDesirabilityColumnFromCategories(stringCol: DG.Column, desirableCategories: string[]):
+ {column: DG.Column, tooltip: string} {
+  const boolArr = new Array<boolean>(stringCol.length);
+  const raw = stringCol.getRawData();
+  const categories = stringCol.categories;
+
+  for (let i = 0; i < stringCol.length; ++i)
+    boolArr[i] = desirableCategories.includes(categories[raw[i]]);
+
+  const nonDesirableCategories = categories.filter((cat) => !desirableCategories.includes(cat));
+
+  const c = `\u2705 ${desirableCategories.join(', ')}`;
+  const unchecked = `\u274c ${nonDesirableCategories.join(', ')}`;
+
+  return {
+    column: DG.Column.fromList(DG.COLUMN_TYPE.BOOL, '', boolArr),
+    tooltip: `Desirability based on the selected categories:\n\n **${c}**\n\n **${unchecked}**`,
+  };
+} // getDesirabilityColumnFromCategories
+
+/** Returns a list of selected categories based on the given list of categories and preferable categories.
+ * @param categories List of categories to select from.
+ * @return List of selected categories.
+*/
+export function getSelectedCategories(categories: string[]): string[] {
+  const selected = categories.filter((cat) => PREFERABLE_CATEGORIES.includes(cat));
+
+  if (selected.length > 0)
+    return selected;
+
+  return [categories[0]];
+}