@datagrok/bio 2.1.4 → 2.1.7

package/package.json

@@ -5,7 +5,7 @@
     "name": "Leonid Stolbov",
     "email": "lstolbov@datagrok.ai"
   },
-  "version": "2.1.4",
+  "version": "2.1.7",
   "description": "Bio is a [package](https://datagrok.ai/help/develop/develop#packages) for the [Datagrok](https://datagrok.ai) platform",
   "repository": {
     "type": "git",
@@ -14,8 +14,8 @@
   },
   "dependencies": {
     "@biowasm/aioli": "^3.1.0",
-    "@datagrok-libraries/bio": "^5.9.15",
-    "@datagrok-libraries/chem-meta": "1.0.1",
+    "@datagrok-libraries/bio": "^5.10.0",
+    "@datagrok-libraries/chem-meta": "^1.0.1",
     "@datagrok-libraries/ml": "^6.2.2",
     "@datagrok-libraries/utils": "^1.15.5",
     "cash-dom": "^8.0.0",

package/scripts/generate_fasta_csv_for_alphabets.R

@@ -4,17 +4,12 @@ library(random)
 alphabetDna <- c('A','C','G','T')
 alphabetRna <- c('A','C','G','U')
 alphabetPt <- c('G', 'L', 'Y', 'S', 'E', 'Q', 'D', 'N', 'F', 'A',
-                'K', 'R', 'H', 'C', 'V', 'P', 'W', 'I', 'M', 'T',)
+                'K', 'R', 'H', 'C', 'V', 'P', 'W', 'I', 'M', 'T')
 
-toAlphabet <- function(v, a;ph){
+toAlphabet <- function(v, alph){
   paste(sapply(v, function(ci){ alph[ci]; }), collapse = '')
 }
 
-mutateString <- function(s, p){
-
-}
-
-seq <- toAlphabet(sample.int(4, 35, replace=TRUE), alphabet);
 seqPt <- toAlphabet(sample.int(20, 35, replace=TRUE), alphabetPt);
 seqDna <- toAlphabet(sample.int(4, 35, replace=TRUE), alphabetDna);
 seqRna <- toAlphabet(sample.int(4, 35, replace=TRUE), alphabetRna);
@@ -60,8 +55,10 @@ seq_mutate <- function(s, p, alphabet){
   res_s;
 }
 
-fastaDna_df <- data.frame(id = 1:100, sequence = sapply(1:100, function(id){ seq_mutate(seqDna, seq_p1, alphabetDna)}));
-write_csv(fastaDna_df, 'D:/HOME/atanas/Datagrok/projs/public/packages/Bio/files/samples/sample_FASTA_DNA.csv');
+for (n in c(100,1000,10000, 100000, 1000000)){
+  fastaDna_df <- data.frame(id = 1:n, sequence = sapply(1:n, function(id){ seq_mutate(seqDna, seq_p1, alphabetDna)}));
+  write_csv(fastaDna_df, sprintf('../files/data/sample_FASTA_DNA-%d.csv', n));
+}
 
 fastaRna_df <- data.frame(id = 1:100, sequence = sapply(1:100, function(id){ seq_mutate(seqRna, seq_p2, alphabetRna)}));
 write_csv(fastaRna_df, 'D:/HOME/atanas/Datagrok/projs/public/packages/Bio/files/samples/sample_FASTA_RNA.csv');

package/src/__jest__/remote.test.ts

@@ -39,33 +39,39 @@ it('TEST', async () => {
   const targetPackage: string = process.env.TARGET_PACKAGE ?? 'Bio';
   console.log(`Testing ${targetPackage} package`);
 
-  const r = await page.evaluate((targetPackage): Promise<object> => {
+  let r = await page.evaluate((targetPackage):Promise<object> => {
     return new Promise<object>((resolve, reject) => {
       (<any>window).grok.functions.eval(targetPackage + ':test()').then((df: any) => {
         const cStatus = df.columns.byName('success');
+        const cSkipped = df.columns.byName('skipped');
         const cMessage = df.columns.byName('result');
         const cCat = df.columns.byName('category');
         const cName = df.columns.byName('name');
         const cTime = df.columns.byName('ms');
         let failed = false;
+        let skipReport = '';
         let passReport = '';
         let failReport = '';
         for (let i = 0; i < df.rowCount; i++) {
           if (cStatus.get(i)) {
-            passReport += `Test result : Success : ${cTime.get(i)} : ` +
-              `${targetPackage}.${cCat.get(i)}.${cName.get(i)} : ${cMessage.get(i)}\n`;
+            if (cSkipped.get(i)) {
+              skipReport += `Test result : Skipped : ${cTime.get(i)} : ${targetPackage}.${cCat.get(i)}.${cName.get(i)} : ${cMessage.get(i)}\n`;
+            } else {
+              passReport += `Test result : Success : ${cTime.get(i)} : ${targetPackage}.${cCat.get(i)}.${cName.get(i)} : ${cMessage.get(i)}\n`;
+            }
           } else {
             failed = true;
-            failReport += `Test result : Failed : ${cTime.get(i)} : ` +
-              `${targetPackage}.${cCat.get(i)}.${cName.get(i)} : ${cMessage.get(i)}\n`;
+            failReport += `Test result : Failed : ${cTime.get(i)} : ${targetPackage}.${cCat.get(i)}.${cName.get(i)} : ${cMessage.get(i)}\n`;
           }
         }
-        resolve({failReport, passReport, failed});
+        resolve({failReport, skipReport, passReport, failed});
       }).catch((e: any) => reject(e));
     });
   }, targetPackage);
   // @ts-ignore
   console.log(r.passReport);
   // @ts-ignore
+  console.log(r.skipReport);
+  // @ts-ignore
   expect(r.failed).checkOutput(false, r.failReport);
-}, 3600000);
+}, 7200000);

package/src/package-test.ts

@@ -3,20 +3,21 @@ import * as DG from 'datagrok-api/dg';
 import {runTests, TestContext, tests} from '@datagrok-libraries/utils/src/test';
 
 import './tests/Palettes-test';
-import './tests/detectors-test';
+import './tests/detectors-tests';
+import './tests/detectors-benchmark-tests';
 import './tests/msa-tests';
 import './tests/sequence-space-test';
 import './tests/activity-cliffs-tests';
 import './tests/splitters-test';
 import './tests/renderers-test';
-import './tests/convert-test';
+import './tests/converters-test';
 import './tests/fasta-handler-test';
 import './tests/fasta-export-tests';
 import './tests/bio-tests';
 import './tests/WebLogo-positions-test';
 import './tests/checkInputColumn-tests';
 import './tests/similarity-diversity-tests';
-import './tests/substructure-filter-tests';
+import './tests/substructure-filters-tests';
 
 export const _package = new DG.Package();
 export {tests};

package/src/package.ts

@@ -11,23 +11,26 @@ import {runKalign, testMSAEnoughMemory} from './utils/multiple-sequence-alignmen
 import {SequenceAlignment, Aligned} from './seq_align';
 import {getEmbeddingColsNames, sequenceSpace, sequenceSpaceByFingerprints} from './analysis/sequence-space';
 import {getActivityCliffs} from '@datagrok-libraries/ml/src/viewers/activity-cliffs';
-import {createLinesGrid, createPropPanelElement, createTooltipElement, getChemSimilaritiesMarix, getSimilaritiesMarix} from './analysis/sequence-activity-cliffs';
-import {createJsonMonomerLibFromSdf, encodeMonomers, getMolfilesFromSeq} from '@datagrok-libraries/bio/src/monomer-works/monomer-utils';
+import {
+  createLinesGrid,
+  createPropPanelElement,
+  createTooltipElement,
+  getChemSimilaritiesMarix,
+  getSimilaritiesMarix
+} from './analysis/sequence-activity-cliffs';
+import {
+  createJsonMonomerLibFromSdf,
+  encodeMonomers,
+  getMolfilesFromSeq
+} from '@datagrok-libraries/bio/src/monomer-works/monomer-utils';
 import {HELM_CORE_LIB_FILENAME} from '@datagrok-libraries/bio/src/utils/const';
 import {getMacroMol} from './utils/atomic-works';
 import {MacromoleculeSequenceCellRenderer} from './utils/cell-renderer';
 import {convert} from './utils/convert';
 import {getMacroMolColumnPropertyPanel, representationsWidget} from './widgets/representations';
-import {MonomerFreqs, TAGS} from '@datagrok-libraries/bio/src/utils/macromolecule';
-import {ALPHABET, NOTATION} from '@datagrok-libraries/bio/src/utils/macromolecule'
 import {_toAtomicLevel} from '@datagrok-libraries/bio/src/monomer-works/to-atomic-level';
 import {FastaFileHandler} from '@datagrok-libraries/bio/src/utils/fasta-handler';
 import {removeEmptyStringRows} from '@datagrok-libraries/utils/src/dataframe-utils';
-import {
-  generateManySequences,
-  generateLongSequence,
-  performanceTest
-} from './tests/test-sequnces-generators';
 
 import {splitAlignedSequences} from '@datagrok-libraries/bio/src/utils/splitter';
 import * as C from './utils/constants';
@@ -36,11 +39,14 @@ import {SequenceDiversityViewer} from './analysis/sequence-diversity-viewer';
 import {invalidateMols, MONOMERIC_COL_TAGS, substructureSearchDialog} from './substructure-search/substructure-search';
 import {saveAsFastaUI} from './utils/save-as-fasta';
 import {BioSubstructureFilter} from './widgets/bio-substructure-filter';
-import { getMonomericMols } from './calculations/monomerLevelMols';
-import { delay } from '@datagrok-libraries/utils/src/test';
+import {getMonomericMols} from './calculations/monomerLevelMols';
+import {delay} from '@datagrok-libraries/utils/src/test';
 import {from, Observable, Subject} from 'rxjs';
-import {Monomer, IMonomerLib, MonomerWorks, MonomerLib, readLibrary,
-        SeqPalette, UnitsHandler, WebLogoViewer, getStats, splitterAsHelm} from '@datagrok-libraries/bio';
+import {
+  TAGS as bio_TAGS,
+  Monomer, IMonomerLib, MonomerWorks, MonomerLib, readLibrary,
+  SeqPalette, UnitsHandler, WebLogoViewer, getStats, splitterAsHelm
+} from '@datagrok-libraries/bio';
 
 const STORAGE_NAME = 'Libraries';
 const LIB_PATH = 'System:AppData/Bio/libraries';
@@ -51,6 +57,7 @@ export let hydrophobPalette: SeqPaletteCustom | null = null;
 
 export class SeqPaletteCustom implements SeqPalette {
   private readonly _palette: { [m: string]: string };
+
   constructor(palette: { [m: string]: string }) {
     this._palette = palette;
   }
@@ -67,7 +74,7 @@ export async function initBio() {
   let logPs: number[] = [];
   const module = await grok.functions.call('Chem:getRdKitModule');
 
-    
+
   const series = monomerLib!.getMonomerMolsByType('PEPTIDE')!;
   Object.keys(series).forEach(symbol => {
     monomers.push(symbol);
@@ -81,7 +88,7 @@ export async function initBio() {
   const sum = logPs.reduce((a, b) => a + b, 0);
   const avg = (sum / logPs.length) || 0;
 
-  let palette: {[monomer: string]: string} = {};
+  let palette: { [monomer: string]: string } = {};
   for (let i = 0; i < monomers.length; i++) {
     palette[monomers[i]] = logPs[i] < avg ? '#4682B4' : '#DC143C';
   }
@@ -134,7 +141,7 @@ export async function libraryPanel(seqColumn: DG.Column): Promise<DG.Widget> {
   let uploadedLibraries: string[] = Object.values(await grok.dapi.userDataStorage.get(STORAGE_NAME, true));
   for (let i = 0; i < uploadedLibraries.length; ++i) {
     let libraryName: string = uploadedLibraries[i];
-    divInputs.append(ui.boolInput(libraryName, true, async() => {
+    divInputs.append(ui.boolInput(libraryName, true, async () => {
       grok.dapi.userDataStorage.remove(STORAGE_NAME, libraryName, true);
       await loadLibraries();
     }).root);
@@ -283,9 +290,9 @@ export async function activityCliffs(df: DG.DataFrame, macroMolecule: DG.Column,
   };
   const tags = {
     'units': macroMolecule.getTag(DG.TAGS.UNITS),
-    'aligned': macroMolecule.getTag(TAGS.aligned),
-    'separator': macroMolecule.getTag(TAGS.separator),
-    'alphabet': macroMolecule.getTag(TAGS.alphabet),
+    'aligned': macroMolecule.getTag(bio_TAGS.aligned),
+    'separator': macroMolecule.getTag(bio_TAGS.separator),
+    'alphabet': macroMolecule.getTag(bio_TAGS.alphabet),
   };
   const sp = await getActivityCliffs(
     df,
@@ -343,7 +350,8 @@ export async function sequenceSpaceTopMenu(table: DG.DataFrame, macroMolecule: D
     return grok.shell
       .tableView(table.name)
       .scatterPlot({x: embedColsNames[0], y: embedColsNames[1], title: 'Sequence space'});
-  };
+  }
+  ;
 
   /*   const encodedCol = encodeMonomers(macroMolecule);
   if (!encodedCol)
@@ -580,7 +588,7 @@ export async function testDetectMacromolecule(path: string): Promise<DG.DataFram
 //tags: panel, bio
 //input: column col {semType: Macromolecule}
 export function splitToMonomers(col: DG.Column<string>): void {
-  if (!col.getTag(TAGS.aligned).includes(C.MSA))
+  if (!col.getTag(bio_TAGS.aligned).includes(C.MSA))
     return grok.shell.error('Splitting is applicable only for aligned sequences');
 
   const tempDf = splitAlignedSequences(col);

package/src/tests/detectors-benchmark-tests.ts

@@ -0,0 +1,165 @@
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+import * as bio from '@datagrok-libraries/bio';
+
+import {after, before, category, test, expect, expectObject} from '@datagrok-libraries/utils/src/test';
+import {UnitsHandler} from '@datagrok-libraries/bio';
+import {Column} from 'datagrok-api/dg';
+
+category('detectorsBenchmark', () => {
+
+  let detectFunc: DG.Func;
+
+  before(async () => {
+    const funcList: DG.Func[] = DG.Func.find({package: 'Bio', name: 'detectMacromolecule'});
+    detectFunc = funcList[0];
+
+    // warm up the detector function
+    const col: DG.Column = DG.Column.fromStrings('seq', ['ACGT', 'ACGT', 'ACGT']);
+    await detectFunc.prepare({col: col}).call();
+  });
+
+  // -- fasta --
+
+  test('fastaDnaShorts50Few50', async () => {
+      const et: number = await detectMacromoleculeBenchmark(10, bio.NOTATION.FASTA, bio.ALPHABET.DNA, 50, 50);
+    },
+    {skipReason: '#1192'});
+
+  test('fastaDnaShorts50Many1E6', async () => {
+      const et: number = await detectMacromoleculeBenchmark(10, bio.NOTATION.FASTA, bio.ALPHABET.DNA, 50, 1E6);
+    },
+    {skipReason: '#1192'});
+
+  test('fastaDnaLong1e6Few50', async () => {
+      const et: number = await detectMacromoleculeBenchmark(10, bio.NOTATION.FASTA, bio.ALPHABET.DNA, 1E6, 50);
+    },
+    {skipReason: '#1192'});
+
+  // -- separator --
+
+  test('separatorDnaShorts50Few50', async () => {
+    const et: number = await detectMacromoleculeBenchmark(10, bio.NOTATION.SEPARATOR, bio.ALPHABET.DNA, 50, 50, '/');
+  });
+
+  test('separatorDnaShorts50Many1E6', async () => {
+      const et: number = await detectMacromoleculeBenchmark(10, bio.NOTATION.SEPARATOR, bio.ALPHABET.DNA, 50, 1E6, '/');
+    },
+    { /* skipReason: 'slow transmit large dataset to detector' */});
+
+  test('separatorDnaLong1e6Few50', async () => {
+      const et: number = await detectMacromoleculeBenchmark(10, bio.NOTATION.SEPARATOR, bio.ALPHABET.DNA, 1E6, 50, '/');
+    },
+    {skipReason: '#1192'});
+
+  async function detectMacromoleculeBenchmark(
+    maxET: number, notation: bio.NOTATION, alphabet: bio.ALPHABET, length: number, count: number, separator?: string
+  ): Promise<number> {
+    return await benchmark<DG.FuncCall, DG.Column>(10,
+      (): DG.FuncCall => {
+        const col: DG.Column = generate(notation, [...bio.getAlphabet(alphabet)], length, count, separator);
+        const funcCall: DG.FuncCall = detectFunc.prepare({col: col});
+        return funcCall;
+      },
+      async (funcCall: DG.FuncCall): Promise<DG.Column> => {
+        return testDetector(funcCall);
+      },
+      (col: DG.Column) => {
+        checkDetectorRes(col, {
+          semType: DG.SEMTYPE.MACROMOLECULE,
+          notation: notation,
+          alphabet: alphabet,
+          separator: separator
+        });
+      });
+  }
+
+  function generate(notation: bio.NOTATION, alphabet: string[], length: number, count: number, separator?: string): DG.Column {
+    let seqMerger: (seqMList: string[], separator?: string) => string;
+
+    switch (notation) {
+    case bio.NOTATION.FASTA:
+      seqMerger = (seqMList: string[]): string => {
+        let res: string = '';
+        for (let j = 0; j < seqMList.length; j++) {
+          const m = seqMList[j];
+          res += m.length == 1 ? m : `[${m}]`;
+        }
+        return res;
+      };
+      break;
+    case bio.NOTATION.SEPARATOR:
+      seqMerger = (seqMList: string[], separator?: string): string => {
+        return seqMList.join(separator);
+      };
+      break;
+    default:
+      throw new Error(`Not supported notation '${notation}'.`);
+    }
+
+    const buildSeq = (alphabet: string[], length: number): string => {
+      const seqMList = new Array<string>(length);
+      for (let j = 0; j < length; j++) {
+        seqMList[j] = alphabet[Math.floor(Math.random() * alphabet.length)];
+      }
+      return seqMerger(seqMList, separator);
+    };
+
+    const seqList: string[] = Array(count);
+    for (let i = 0; i < count; i++) {
+      seqList[i] = buildSeq(alphabet, length);
+    }
+
+    return DG.Column.fromStrings('seq', seqList);
+  }
+
+  type TgtType = { semType: string, notation: bio.NOTATION, alphabet: bio.ALPHABET, separator?: string };
+
+  function testDetector(funcCall: DG.FuncCall): DG.Column {
+    //const semType: string = await grok.functions.call('Bio:detectMacromolecule', {col: col});
+    funcCall.callSync();
+    const semType = funcCall.getOutputParamValue();
+
+    const col: DG.Column = funcCall.inputs.col;
+    if (semType) col.semType = semType;
+    return col;
+  }
+
+  function checkDetectorRes(col: DG.Column, tgt: TgtType): void {
+    const uh = new UnitsHandler(col);
+    expect(col.semType, tgt.semType);
+    expect(uh.notation, tgt.notation);
+    expect(uh.alphabet, tgt.alphabet);
+    expect(uh.separator, tgt.separator);
+  }
+
+});
+
+
+/** Returns ET [ms] of test() */
+async function benchmark<TData, TRes>(
+  maxET: number, prepare: () => TData, test: (data: TData) => Promise<TRes>, check: (res: TRes) => void
+): Promise<number> {
+  const data: TData = prepare();
+
+  const t1: number = Date.now();
+  // console.profile();
+  const res: TRes = await test(data);
+  //console.profileEnd();
+  const t2: number = Date.now();
+
+  check(res);
+
+  const resET: number = t2 - t1;
+  if (resET > maxET) {
+    const errMsg = `ET ${resET} ms is more than max allowed ${maxET} ms.`;
+    console.error(errMsg);
+    throw new Error(errMsg);
+  } else {
+    console.log(`ET ${resET} ms is OK.`);
+  }
+
+  return resET;
+}
+

package/src/tests/renderers-test.ts

@@ -7,7 +7,7 @@ import {after, before, category, delay, expect, test} from '@datagrok-libraries/
 import {importFasta, multipleSequenceAlignmentAny} from '../package';
 import {convertDo} from '../utils/convert';
 import {SEM_TYPES, TAGS} from '../utils/constants';
-import {generateLongSequence, generateManySequences, performanceTest} from './test-sequnces-generators';
+import {generateLongSequence, generateManySequences, performanceTest} from './utils/sequences-generators';
 import {errorToConsole} from '@datagrok-libraries/utils/src/to-console';
 
 category('renderers', () => {
@@ -17,11 +17,6 @@ category('renderers', () => {
   before(async () => {
     tvList = [];
     dfList = [];
-    await grok.functions.call('Bio:initBio')
-      .catch((err) => {
-        console.error(errorToConsole(err));
-        throw err;
-      });
   });
 
   after(async () => {

package/src/tests/splitters-test.ts

@@ -16,11 +16,6 @@ category('splitters', () => {
   before(async () => {
     tvList = [];
     dfList = [];
-    await grok.functions.call('Bio:initBio')
-      .catch((err) => {
-        console.error(errorToConsole(err));
-        throw err;
-      });
   });
 
   after(async () => {

package/src/tests/{substructure-filter-tests.ts → substructure-filters-tests.ts}

@@ -69,5 +69,5 @@ category('substructureFilters', async () => {
     expect(filter.dataFrame!.filter.trueCount, 1);
     expect(filter.dataFrame!.filter.get(3), true);
     helmTableView.close();
-  });
+  }, {skipReason: '#1206'});
 });