@datagrok/bio 2.11.3 → 2.11.5

package/package.json

@@ -5,7 +5,7 @@
     "name": "Leonid Stolbov",
     "email": "lstolbov@datagrok.ai"
   },
-  "version": "2.11.3",
+  "version": "2.11.5",
   "description": "Bioinformatics support (import/export of sequences, conversion, visualization, analysis). [See more](https://github.com/datagrok-ai/public/blob/master/packages/Bio/README.md) for details.",
   "repository": {
     "type": "git",
@@ -34,9 +34,9 @@
   ],
   "dependencies": {
     "@biowasm/aioli": "^3.1.0",
-    "@datagrok-libraries/bio": "^5.39.0",
+    "@datagrok-libraries/bio": "^5.39.1",
     "@datagrok-libraries/chem-meta": "^1.0.1",
-    "@datagrok-libraries/ml": "^6.3.51",
+    "@datagrok-libraries/ml": "^6.3.53",
     "@datagrok-libraries/tutorials": "^1.3.6",
     "@datagrok-libraries/utils": "^4.0.17",
     "cash-dom": "^8.0.0",

package/src/analysis/sequence-space.ts

@@ -55,21 +55,19 @@ export async function sequenceSpaceByFingerprints(spaceParams: ISequenceSpacePar
   return result;
 }
 
-export async function getSequenceSpace(spaceParams: ISequenceSpaceParams,
-  progressFunc?: (epochNum: number, epochsLength: number, embedding: number[][]) => void
-): Promise<ISequenceSpaceResult> {
-  const ncUH = UnitsHandler.getOrCreate(spaceParams.seqCol);
-
-  //const distanceFName = ncUH.isMsa() ? MmDistanceFunctionsNames.HAMMING : MmDistanceFunctionsNames.LEVENSHTEIN;
-  const seqList = spaceParams.seqCol.toList();
-
+export async function getEncodedSeqSpaceCol(
+  seqCol: DG.Column, similarityMetric: BitArrayMetrics | MmDistanceFunctionsNames
+): Promise<{seqList:string[], options: {[_:string]: any}}> {
+// encodes sequences using utf charachters to also support multichar and non fasta sequences
+  const ncUH = UnitsHandler.getOrCreate(seqCol);
+  const seqList = seqCol.toList();
   const splitter = ncUH.getSplitter();
   const seqColLength = seqList.length;
   let charCodeCounter = 36;
   const charCodeMap = new Map<string, string>();
   for (let i = 0; i < seqColLength; i++) {
     const seq = seqList[i];
-    if (seqList[i] === null || spaceParams.seqCol.isNone(i)) {
+    if (seqList[i] === null || seqCol.isNone(i)) {
       seqList[i] = null;
       continue;
     }
@@ -84,8 +82,8 @@ export async function getSequenceSpace(spaceParams: ISequenceSpaceParams,
       seqList[i] += charCodeMap.get(char)!;
     }
   }
-
-  if (spaceParams.similarityMetric === MmDistanceFunctionsNames.MONOMER_CHEMICAL_DISTANCE) {
+  let options = {};
+  if (similarityMetric === MmDistanceFunctionsNames.MONOMER_CHEMICAL_DISTANCE) {
     const monomers = Array.from(charCodeMap.keys());
     const monomerRes = await calculateMonomerSimilarity(monomers);
     // the susbstitution matrix contains similarity, but we need distances
@@ -98,10 +96,34 @@ export async function getSequenceSpace(spaceParams: ISequenceSpaceParams,
     Object.entries(monomerRes.alphabetIndexes).forEach(([key, value]) => {
       monomerHashToMatrixMap[charCodeMap.get(key)!] = value;
     });
-    spaceParams.options.distanceFnArgs = {scoringMatrix: monomerRes.scoringMatrix,
+    // sets distance function args in place.
+    options = {scoringMatrix: monomerRes.scoringMatrix,
       alphabetIndexes: monomerHashToMatrixMap} satisfies mmDistanceFunctionArgs;
   }
+  // else if (similarityMetric === MmDistanceFunctionsNames.NEEDLEMANN_WUNSCH) {
+  //   const alphabetIndexes: any = {};
+  //   let i = 0;
+  //   charCodeMap.forEach((value) => {
+  //     alphabetIndexes[value] = i;
+  //     i++;
+  //   });
+  //   options = {alphabetIndexes};
+  // }
+  return {seqList, options};
+}
+
+export async function getSequenceSpace(spaceParams: ISequenceSpaceParams,
+  progressFunc?: (epochNum: number, epochsLength: number, embedding: number[][]) => void
+): Promise<ISequenceSpaceResult> {
+  const ncUH = UnitsHandler.getOrCreate(spaceParams.seqCol);
+  if (ncUH.isHelm())
+    return await sequenceSpaceByFingerprints(spaceParams);
+
+
+  const {seqList, options} = await getEncodedSeqSpaceCol(spaceParams.seqCol, spaceParams.similarityMetric);
 
+  spaceParams.options = spaceParams.options ?? {};
+  spaceParams.options.distanceFnArgs = options;
   const sequenceSpaceResult = await reduceDimensinalityWithNormalization(
     seqList,
     spaceParams.methodName,

package/src/demo/bio01b-hierarchical-clustering-and-activity-cliffs.ts

@@ -13,6 +13,7 @@ import {getDendrogramService, IDendrogramService} from '@datagrok-libraries/bio/
 import {handleError} from './utils';
 import {DemoScript} from '@datagrok-libraries/tutorials/src/demo-script';
 import {DimReductionMethods} from '@datagrok-libraries/ml/src/reduce-dimensionality';
+import {MmDistanceFunctionsNames} from '@datagrok-libraries/ml/src/macromolecule-distance-functions';
 
 const dataFn: string = 'data/sample_FASTA_PT_activity.csv';
 
@@ -53,7 +54,7 @@ export async function demoBio01bUI() {
       .step('Find activity cliffs', async () => {
         activityCliffsViewer = (await activityCliffs(
           df, df.getCol('Sequence'), df.getCol('Activity'),
-          80, dimRedMethod)) as DG.ScatterPlotViewer;
+          80, dimRedMethod, MmDistanceFunctionsNames.LEVENSHTEIN)) as DG.ScatterPlotViewer;
         view.dockManager.dock(activityCliffsViewer, DG.DOCK_TYPE.RIGHT, null, 'Activity Cliffs', 0.35);
 
         // Show grid viewer with the cliffs

package/src/package.ts

@@ -11,10 +11,10 @@ import {DimReductionMethods, ITSNEOptions, IUMAPOptions} from '@datagrok-librari
 import {SequenceSpaceFunctionEditor} from '@datagrok-libraries/ml/src/functionEditors/seq-space-editor';
 import {ActivityCliffsFunctionEditor} from '@datagrok-libraries/ml/src/functionEditors/activity-cliffs-editor';
 import {
-  ISequenceSpaceParams, getActivityCliffs, SequenceSpaceFunc
+  ISequenceSpaceParams, getActivityCliffs, SequenceSpaceFunc, CLIFFS_COL_ENCODE_FN
 } from '@datagrok-libraries/ml/src/viewers/activity-cliffs';
 import {MmDistanceFunctionsNames} from '@datagrok-libraries/ml/src/macromolecule-distance-functions';
-import {BitArrayMetrics, BitArrayMetricsNames} from '@datagrok-libraries/ml/src/typed-metrics';
+import {BitArrayMetrics} from '@datagrok-libraries/ml/src/typed-metrics';
 import {
   TAGS as bioTAGS, ALPHABET, NOTATION,
 } from '@datagrok-libraries/bio/src/utils/macromolecule';
@@ -35,14 +35,14 @@ import {
 import {VdRegionsViewer} from './viewers/vd-regions-viewer';
 import {SequenceAlignment} from './seq_align';
 import {
-  ISequenceSpaceResult, getEmbeddingColsNames, getSequenceSpace, sequenceSpaceByFingerprints
+  ISequenceSpaceResult, getEmbeddingColsNames, getEncodedSeqSpaceCol, getSequenceSpace, sequenceSpaceByFingerprints
 } from './analysis/sequence-space';
 import {
   createLinesGrid, createPropPanelElement, createTooltipElement, getChemSimilaritiesMatrix,
 } from './analysis/sequence-activity-cliffs';
 import {SequenceSimilarityViewer} from './analysis/sequence-similarity-viewer';
 import {SequenceDiversityViewer} from './analysis/sequence-diversity-viewer';
-import {SubstructureSearchDialog} from './substructure-search/substructure-search';
+import {MONOMERIC_COL_TAGS, SubstructureSearchDialog, invalidateMols} from './substructure-search/substructure-search';
 import {convert} from './utils/convert';
 import {getMacromoleculeColumnPropertyPanel} from './widgets/representations';
 import {saveAsFastaUI} from './utils/save-as-fasta';
@@ -79,6 +79,7 @@ import {errInfo} from './utils/err-info';
 import {SHOW_SCATTERPLOT_PROGRESS} from '@datagrok-libraries/ml/src/functionEditors/seq-space-base-editor';
 import {DIMENSIONALITY_REDUCER_TERMINATE_EVENT}
   from '@datagrok-libraries/ml/src/workers/dimensionality-reducing-worker-creator';
+import BitArray from '@datagrok-libraries/utils/src/bit-array';
 
 export const _package = new BioPackage();
 
@@ -389,12 +390,14 @@ export async function getRegionTopMenu(
 //input: column activities
 //input: double similarity = 80 [Similarity cutoff]
 //input: string methodName { choices:["UMAP", "t-SNE"] }
+//input: string similarityMetric { choices:["Hamming", "Levenshtein", "Monomer chemical distance"] }
 //input: object options {optional: true}
 //output: viewer result
 //editor: Bio:SeqActivityCliffsEditor
 export async function activityCliffs(df: DG.DataFrame, macroMolecule: DG.Column<string>, activities: DG.Column,
-  similarity: number, methodName: DimReductionMethods, options?: (IUMAPOptions | ITSNEOptions) & Options,
-): Promise<DG.Viewer | undefined> {
+  similarity: number, methodName: DimReductionMethods,
+  similarityMetric: MmDistanceFunctionsNames | BitArrayMetrics,
+  options?: (IUMAPOptions | ITSNEOptions) & Options): Promise<DG.Viewer | undefined> {
   if (!checkInputColumnUI(macroMolecule, 'Activity Cliffs'))
     return;
   const axesNames = getEmbeddingColsNames(df);
@@ -404,21 +407,27 @@ export async function activityCliffs(df: DG.DataFrame, macroMolecule: DG.Column<
     'separator': macroMolecule.getTag(bioTAGS.separator),
     'alphabet': macroMolecule.getTag(bioTAGS.alphabet),
   };
+  let cliffsEncodeFunction: (seqCol: DG.Column, similarityMetric: MmDistanceFunctionsNames | BitArrayMetrics) => any =
+     getEncodedSeqSpaceCol;
   const ncUH = UnitsHandler.getOrCreate(macroMolecule);
-  let columnDistanceMetric: BitArrayMetrics | MmDistanceFunctionsNames = BitArrayMetricsNames.Tanimoto;
-  let seqCol = macroMolecule;
-  let sequenceSpaceFunc: SequenceSpaceFunc = sequenceSpaceByFingerprints;
-  if (ncUH.isFasta() || (ncUH.isSeparator() && ncUH.alphabet && ncUH.alphabet !== ALPHABET.UN)) {
-    if (ncUH.isFasta()) {
-      columnDistanceMetric = ncUH.getDistanceFunctionName();
-    } else {
-      seqCol = ncUH.convert(NOTATION.FASTA);
-      const uh = UnitsHandler.getOrCreate(seqCol);
-      columnDistanceMetric = uh.getDistanceFunctionName();
-      tags.units = NOTATION.FASTA;
-    }
-    sequenceSpaceFunc = getSequenceSpace;
+  const columnDistanceMetric: MmDistanceFunctionsNames | BitArrayMetrics = similarityMetric;
+  const seqCol = macroMolecule;
+
+  let sequenceSpaceFunc: SequenceSpaceFunc = getSequenceSpace;
+  if (ncUH.isHelm()) {
+    sequenceSpaceFunc = sequenceSpaceByFingerprints;
+    cliffsEncodeFunction = async (seqCol: DG.Column, similarityMetric: MmDistanceFunctionsNames | BitArrayMetrics) => {
+      await invalidateMols(seqCol, false);
+      const molecularCol = seqCol.temp[MONOMERIC_COL_TAGS.MONOMERIC_MOLS];
+      const fingerPrints: DG.Column =
+        await grok.functions.call('Chem:getMorganFingerprints', {molColumn: molecularCol});
+      const fingerPrintsBitArray = fingerPrints.toList().map((f: DG.BitSet) =>
+        BitArray.fromUint32Array(f.length, new Uint32Array(f.getBuffer().buffer)));
+      return {seqList: fingerPrintsBitArray, options: {}};
+    };
   }
+
+
   const runCliffs = async () => {
     const sp = await getActivityCliffs(
       df,
@@ -437,25 +446,26 @@ export async function activityCliffs(df: DG.DataFrame, macroMolecule: DG.Column<
       createTooltipElement,
       createPropPanelElement,
       createLinesGrid,
-      options);
+      {...(options ?? {}), [CLIFFS_COL_ENCODE_FN]: cliffsEncodeFunction});
     return sp;
   };
 
-  const allowedRowCount = 20000;
-  const fastRowCount = methodName === DimReductionMethods.UMAP ? 5000 : 2000;
+  const allowedRowCount = methodName === DimReductionMethods.UMAP ? 200_000 : 20_000;
+  const fastRowCount = methodName === DimReductionMethods.UMAP ? 5_000 : 2_000;
   if (df.rowCount > allowedRowCount) {
     grok.shell.warning(`Too many rows, maximum for sequence activity cliffs is ${allowedRowCount}`);
     return;
   }
 
-  return new Promise<DG.Viewer>((resolve, reject) => {
+  return new Promise<DG.Viewer | undefined>((resolve, reject) => {
     if (df.rowCount > fastRowCount && !options?.[BYPASS_LARGE_DATA_WARNING]) {
       ui.dialog().add(ui.divText(`Activity cliffs analysis might take several minutes.
     Do you want to continue?`))
         .onOK(async () => {
-          const progressBar = DG.TaskBarProgressIndicator.create(`Running sequence activity cliffs ...`);
-          runCliffs().then((res) => resolve(res)).catch((err) => reject(err)).finally(() => { progressBar.close(); });
+          //const progressBar = DG.TaskBarProgressIndicator.create(`Running sequence activity cliffs ...`);
+          runCliffs().then((res) => resolve(res)).catch((err) => reject(err)).finally(() => {});
         })
+        .onCancel(() => { resolve(undefined); })
         .show();
     } else {
       runCliffs().then((res) => resolve(res)).catch((err) => reject(err));
@@ -525,7 +535,7 @@ export async function sequenceSpaceTopMenu(
       methodName: methodName,
       similarityMetric: similarityMetric,
       embedAxesNames: embedColsNames,
-      options: {...options, sparseMatrixThreshold: sparseMatrixThreshold ?? 0.8,
+      options: {...options, sparseMatrixThreshold: sparseMatrixThreshold ?? 0.5,
         usingSparseMatrix: table.rowCount > 20000},
     };
 
@@ -1084,10 +1094,10 @@ export async function demoBioHelmMsaSequenceSpace(): Promise<void> {
   await demoBio05UI();
 }
 
-//name: enumeratorColumnChoice
+//name: polyToolColumnChoice
 //input: dataframe df [Input data table]
 //input: column macroMolecule
-export async function enumeratorColumnChoice(df: DG.DataFrame, macroMolecule: DG.Column): Promise<void> {
+export async function polyToolColumnChoice(df: DG.DataFrame, macroMolecule: DG.Column): Promise<void> {
   _setPeptideColumn(macroMolecule);
   await grok.data.detectSemanticTypes(df);
 }

package/src/tests/activity-cliffs-tests.ts

@@ -8,6 +8,8 @@ import {_testActivityCliffsOpen} from './activity-cliffs-utils';
 import {DimReductionMethods} from '@datagrok-libraries/ml/src/reduce-dimensionality';
 
 import {_package} from '../package-test';
+import {MmDistanceFunctionsNames} from '@datagrok-libraries/ml/src/macromolecule-distance-functions';
+import {BitArrayMetricsNames} from '@datagrok-libraries/ml/src/typed-metrics';
 
 
 category('activityCliffs', async () => {
@@ -39,7 +41,7 @@ category('activityCliffs', async () => {
     const cliffsNum = DG.Test.isInBenchmark ? 6 : 3;
 
     await _testActivityCliffsOpen(actCliffsDf, DimReductionMethods.UMAP,
-      'sequence', 'Activity', 90, cliffsNum);
+      'sequence', 'Activity', 90, cliffsNum, MmDistanceFunctionsNames.LEVENSHTEIN);
   });
 
   test('activityCliffsWithEmptyRows', async () => {
@@ -49,7 +51,7 @@ category('activityCliffs', async () => {
     viewList.push(actCliffsTableViewWithEmptyRows);
 
     await _testActivityCliffsOpen(actCliffsDfWithEmptyRows, DimReductionMethods.UMAP,
-      'sequence', 'Activity', 90, 3);
+      'sequence', 'Activity', 90, 3, MmDistanceFunctionsNames.LEVENSHTEIN);
   });
 
   test('Helm', async () => {
@@ -57,6 +59,6 @@ category('activityCliffs', async () => {
     const view = grok.shell.addTableView(df);
 
     await _testActivityCliffsOpen(df, DimReductionMethods.UMAP,
-      'HELM', 'Activity', 90, 53);
+      'HELM', 'Activity', 90, 53, BitArrayMetricsNames.Tanimoto);
   });
 });

package/src/tests/activity-cliffs-utils.ts

@@ -4,14 +4,17 @@ import * as grok from 'datagrok-api/grok';
 import {expect} from '@datagrok-libraries/utils/src/test';
 import {activityCliffs, BYPASS_LARGE_DATA_WARNING} from '../package';
 import {DimReductionMethods} from '@datagrok-libraries/ml/src/reduce-dimensionality';
+import {MmDistanceFunctionsNames} from '@datagrok-libraries/ml/src/macromolecule-distance-functions';
+import {BitArrayMetrics} from '@datagrok-libraries/ml/src/typed-metrics';
 
 export async function _testActivityCliffsOpen(df: DG.DataFrame, drMethod: DimReductionMethods,
-  seqColName: string, activityColName: string, similarityThr: number, tgtNumberCliffs: number
+  seqColName: string, activityColName: string, similarityThr: number, tgtNumberCliffs: number,
+  similarityMetric: MmDistanceFunctionsNames | BitArrayMetrics
 ): Promise<void> {
   await grok.data.detectSemanticTypes(df);
   const scatterPlot = await activityCliffs(
     df, df.getCol(seqColName), df.getCol(activityColName),
-    similarityThr, drMethod, {[`${BYPASS_LARGE_DATA_WARNING}`]: true});
+    similarityThr, drMethod, similarityMetric, {[`${BYPASS_LARGE_DATA_WARNING}`]: true});
   // const scatterPlot = (await grok.functions.call('Bio:activityCliffs', {
   //   table: df, molecules: df.getCol(colName), activities: df.getCol('Activity'),
   //   similarity: 50, methodName: method

package/src/utils/monomer-lib.ts

@@ -8,9 +8,7 @@ import {IMonomerLib, Monomer} from '@datagrok-libraries/bio/src/types/index';
 import {MolfileHandler} from '@datagrok-libraries/chem-meta/src/parsing-utils/molfile-handler';
 import {
   createJsonMonomerLibFromSdf,
-  getJsonMonomerLibForEnumerator,
   IMonomerLibHelper,
-  isValidEnumeratorLib,
 } from '@datagrok-libraries/bio/src/monomer-works/monomer-utils';
 import {
   HELM_REQUIRED_FIELDS as REQ, HELM_OPTIONAL_FIELDS as OPT, HELM_POLYMER_TYPE
@@ -18,12 +16,8 @@ import {
 
 import {_package} from '../package';
 
-const _HELM_REQUIRED_FIELDS_ARRAY = [
-  REQ.SYMBOL, REQ.NAME, REQ.MOLFILE, REQ.AUTHOR, REQ.ID,
-  REQ.RGROUPS, REQ.SMILES, REQ.POLYMER_TYPE, REQ.MONOMER_TYPE, REQ.CREATE_DATE,
-] as const;
+import {PolyToolMonomerLibHandler} from '@datagrok-libraries/bio/src/utils/poly-tool/monomer-lib-handler';
 
-const _HELM_OPTIONAL_FIELDS_ARRAY = [OPT.NATURAL_ANALOG, OPT.META] as const;
 // -- Monomer libraries --
 export const LIB_STORAGE_NAME = 'Libraries';
 export const LIB_PATH = 'System:AppData/Bio/libraries/';
@@ -291,8 +285,9 @@ export class MonomerLibHelper implements IMonomerLibHelper {
       }
       const df = await fileSource.readCsv(fileName);
       const json = toJson(df);
-      if (isValidEnumeratorLib(json))
-        rawLibData = getJsonMonomerLibForEnumerator(json);
+      const polyToolMonomerLib = new PolyToolMonomerLibHandler(json);
+      if (polyToolMonomerLib.isValid())
+        rawLibData = polyToolMonomerLib.getJsonMonomerLib();
       else
         throw new Error('Invalid format of CSV monomer lib');
     } else {