@datagrok/bio 1.4.2 → 1.5.3

package/src/tests/detectors-test.ts

@@ -7,6 +7,8 @@ import * as DG from 'datagrok-api/dg';
 import {mmSemType} from '../const';
 import {importFasta} from '../package';
 
+type DfReaderFunc = () => Promise<DG.DataFrame>;
+
 category('detectors', () => {
   const csvDf1: string = `col1
 1
@@ -87,36 +89,126 @@ YNR-WYV-KHP
 MWRSWY-CKHP
 `;
 
-  test('testDetectorsNegative1', async () => { await _testDetectorsNegative(csvDf1); });
-  test('testDetectorsNegative2', async () => { await _testDetectorsNegative(csvDf2); });
-  test('testDetectorsNegative3', async () => { await _testDetectorsNegative(csvDf3); });
-  test('testDetectorsNegativeSmiles', async () => { await _testDetectorsNegative(csvDfSmiles); });
-
-  test('testDetectorsN1', async () => { await _testDetectorsN1(csvDfN1); });
-  test('testDetectorsAA1', async () => { await _testDetectorsAA1(csvDfAA1); });
-  test('testDetectorsMsaN1', async () => { await _testDetectorsMsaN1(csvDfMsaN1); });
-  test('testDetectorsMsaAA1', async () => { await _testDetectorsMsaAA1(csvDfMsaAA1); });
-
-  test('testDetectorsSepNt', async () => { await _testDetectorsSepNt(csvDfSepNt, '*'); });
-  test('testDetectorsSepPt', async () => { await _testDetectorsSepPt(csvDfSepPt, '-'); });
-  test('testDetectorsSepUn1', async () => { await _testDetectorsSepUn(csvDfSepUn1, '-'); });
-  test('testDetectorsSepUn2', async () => { await _testDetectorsSepUn(csvDfSepUn2, '/'); });
+  const enum Samples {
+    peptidesComplex = 'PeptidesComplex',
+    fastaCsv = 'FastaCsv',
+    msaComplex = 'MsaComplex',
+    idCsv = 'IdCsv',
+  }
+
+  const samples: { [key: string]: string } = {
+    'PeptidesComplex': 'System:AppData/Bio/samples/peptides_complex_msa.csv',
+    'FastaCsv': 'System:AppData/Bio/samples/sample_FASTA.csv',
+    'MsaComplex': 'System:AppData/Bio/samples/sample_MSA.csv',
+    'IdCsv': 'System:AppData/Bio/samples/id.csv',
+  };
+
+  const _samplesDfs: { [key: string]: Promise<DG.DataFrame> } = {};
+  const readSamplesCsv: (key: string) => DfReaderFunc = (key: string) => {
+    return async () => {
+      if (!(key in _samplesDfs)) {
+        _samplesDfs[key] = (async (): Promise<DG.DataFrame> => {
+          const csv: string = await grok.dapi.files.readAsText(samples[key]);
+          const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
+          await grok.data.detectSemanticTypes(df);
+          return df;
+        })();
+      }
+      return _samplesDfs[key];
+    };
+  };
+
+  const _csvDfs: { [key: string]: Promise<DG.DataFrame> } = {};
+  const readCsv: (key: string, csv: string) => DfReaderFunc = (key: string, csv: string) => {
+    return async () => {
+      if (!(key in _csvDfs)) {
+        _csvDfs[key] = (async (): Promise<DG.DataFrame> => {
+          const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
+          await grok.data.detectSemanticTypes(df);
+          return df;
+        })();
+      }
+      return _csvDfs[key];
+    };
+  };
+
+  test('Negative1', async () => { await _testNeg(readCsv('csvDf1', csvDf1), 'col1'); });
+  test('Negative2', async () => { await _testNeg(readCsv('csvDf2', csvDf2), 'col1'); });
+  test('Negative3', async () => { await _testNeg(readCsv('csvDf3', csvDf3), 'col1'); });
+  test('NegativeSmiles', async () => { await _testNeg(readCsv('csvDfSmiles', csvDfSmiles), 'col1'); });
+
+  test('N1', async () => { await _testN1(csvDfN1); });
+  test('AA1', async () => { await _testAA1(csvDfAA1); });
+  test('MsaN1', async () => { await _testMsaN1(csvDfMsaN1); });
+  test('MsaAA1', async () => { await _testMsaAA1(csvDfMsaAA1); });
+
+  test('SepNt', async () => { await _testSepNt(csvDfSepNt, '*'); });
+  test('SepPt', async () => { await _testSepPt(csvDfSepPt, '-'); });
+  test('SepUn1', async () => { await _testSepUn(csvDfSepUn1, '-'); });
+  test('SepUn2', async () => { await _testSepUn(csvDfSepUn2, '/'); });
+
+  test('SepMsaN1', async () => { await _testSepMsaN1(csvDfSepMsaN1); });
+
+  test('SamplesFastaCsvPt', async () => {
+    await _testSamplesFastaCsvPt();
+  });
+  test('SamplesFastaCsvNegativeEntry', async () => {
+    await _testNeg(readSamplesCsv(Samples.fastaCsv), 'Entry');
+  });
+  test('SamplesFastaCsvNegativeLength', async () => {
+    await _testNeg(readSamplesCsv(Samples.fastaCsv), 'Length');
+  });
+  test('SamplesFastaCsvNegativeUniProtKB', async () => {
+    await _testNeg(readSamplesCsv(Samples.fastaCsv), 'UniProtKB');
+  });
+
+  test('SamplesFastaFastaPt', async () => { await _testSamplesFastaFastaPt(); });
+
+  // System:AppData/Bio/samples/peptides_complex_align.csv contains monomers with spaces
+  // test('SamplesPeptidesComplexUn', async () => {
+  //   await _testSamplesPeptidesComplexUn();
+  // });
+
+  test('samplesPeptidesComplexNegativeID', async () => {
+    await _testNeg(readSamplesCsv(Samples.peptidesComplex), 'ID');
+  });
+  test('SamplesPeptidesComplexNegativeMeasured', async () => {
+    await _testNeg(readSamplesCsv(Samples.peptidesComplex), 'Measured');
+  });
+  test('SamplesPeptidesComplexNegativeValue', async () => {
+    await _testNeg(readSamplesCsv(Samples.peptidesComplex), 'Value');
+  });
+
+  test('samplesMsaComplexUn', async () => {
+    await _testPos(readSamplesCsv(Samples.msaComplex), 'MSA', 'separator:SEQ.MSA:UN', '/');
+  });
+  test('samplesMsaComplexNegativeActivity', async () => {
+    await _testNeg(readSamplesCsv(Samples.msaComplex), 'Activity');
+  });
+
+  test('samplesIdCsvNegativeID', async () => {
+    await _testNeg(readSamplesCsv(Samples.idCsv), 'ID');
+  });
+});
 
-  test('testDetectorsSepMsaN1', async () => { await _testDetectorsSepMsaN1(csvDfSepMsaN1); });
+export async function _testNeg(readDf: DfReaderFunc, colName: string) {
+  const df: DG.DataFrame = await readDf();
 
-  test('testDetectorsSamplesFastaCsvPt', async () => { await _testDetectorsSamplesFastaCsvPt(); });
-  test('testDetectorsSamplesFastaFastaPt', async () => { await _testDetectorsSamplesFastaFastaPt(); });
-});
+  const col: DG.Column = df.col(colName)!;
+  expect(col.semType === mmSemType, false);
+}
 
-export async function _testDetectorsNegative(csvDf: string) {
-  const df: DG.DataFrame = DG.DataFrame.fromCsv(csvDf);
-  await grok.data.detectSemanticTypes(df);
+export async function _testPos(readDf: DfReaderFunc, colName: string, units: string, separator: string) {
+  const df: DG.DataFrame = await readDf();
 
-  const col1: DG.Column = df.col('col1')!;
-  expect(col1.semType == mmSemType, false);
+  const col: DG.Column = df.col(colName)!;
+  expect(col.semType === mmSemType, true);
+  expect(col.getTag(DG.TAGS.UNITS), units);
+  if (separator)
+    expect(col.getTag('separator'), separator);
 }
 
-export async function _testDetectorsN1(csvDfN1: string) {
+export async function _testN1(csvDfN1: string) {
   const dfN1: DG.DataFrame = DG.DataFrame.fromCsv(csvDfN1);
   await grok.data.detectSemanticTypes(dfN1);
 
@@ -125,7 +217,7 @@ export async function _testDetectorsN1(csvDfN1: string) {
   expect(col.getTag(DG.TAGS.UNITS), 'fasta:SEQ:NT');
 }
 
-export async function _testDetectorsAA1(csvDfAA1: string) {
+export async function _testAA1(csvDfAA1: string) {
   const dfAA1: DG.DataFrame = DG.DataFrame.fromCsv(csvDfAA1);
   await grok.data.detectSemanticTypes(dfAA1);
 
@@ -134,7 +226,7 @@ export async function _testDetectorsAA1(csvDfAA1: string) {
   expect(col.getTag(DG.TAGS.UNITS), 'fasta:SEQ:PT');
 }
 
-export async function _testDetectorsMsaN1(csvDfMsaN1: string) {
+export async function _testMsaN1(csvDfMsaN1: string) {
   const dfMsaN1: DG.DataFrame = DG.DataFrame.fromCsv(csvDfMsaN1);
   await grok.data.detectSemanticTypes(dfMsaN1);
 
@@ -143,7 +235,7 @@ export async function _testDetectorsMsaN1(csvDfMsaN1: string) {
   expect(col.getTag(DG.TAGS.UNITS), 'fasta:SEQ.MSA:NT');
 }
 
-export async function _testDetectorsMsaAA1(csvDfMsaAA1: string) {
+export async function _testMsaAA1(csvDfMsaAA1: string) {
   const dfMsaAA1: DG.DataFrame = DG.DataFrame.fromCsv(csvDfMsaAA1);
   await grok.data.detectSemanticTypes(dfMsaAA1);
 
@@ -152,7 +244,7 @@ export async function _testDetectorsMsaAA1(csvDfMsaAA1: string) {
   expect(col.getTag(DG.TAGS.UNITS), 'fasta:SEQ.MSA:PT');
 }
 
-export async function _testDetectorsSepNt(csv: string, separator: string) {
+export async function _testSepNt(csv: string, separator: string) {
   const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
   await grok.data.detectSemanticTypes(df);
 
@@ -162,7 +254,7 @@ export async function _testDetectorsSepNt(csv: string, separator: string) {
   expect(col.getTag('separator'), separator);
 }
 
-export async function _testDetectorsSepPt(csv: string, separator: string) {
+export async function _testSepPt(csv: string, separator: string) {
   const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
   await grok.data.detectSemanticTypes(df);
 
@@ -172,7 +264,7 @@ export async function _testDetectorsSepPt(csv: string, separator: string) {
   expect(col.getTag('separator'), separator);
 }
 
-export async function _testDetectorsSepUn(csv: string, separator: string) {
+export async function _testSepUn(csv: string, separator: string) {
   const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
   await grok.data.detectSemanticTypes(df);
 
@@ -182,7 +274,7 @@ export async function _testDetectorsSepUn(csv: string, separator: string) {
   expect(col.getTag('separator'), separator);
 }
 
-export async function _testDetectorsSepMsaN1(csvDfSepMsaN1: string) {
+export async function _testSepMsaN1(csvDfSepMsaN1: string) {
   const dfSepMsaN1: DG.DataFrame = DG.DataFrame.fromCsv(csvDfSepMsaN1);
   await grok.data.detectSemanticTypes(dfSepMsaN1);
 
@@ -191,7 +283,7 @@ export async function _testDetectorsSepMsaN1(csvDfSepMsaN1: string) {
   expect(col.getTag(DG.TAGS.UNITS), 'separator:SEQ.MSA:NT');
 }
 
-export async function _testDetectorsSamplesFastaCsvPt() {
+export async function _testSamplesFastaCsvPt() {
   const csv: string = await grok.dapi.files.readAsText('System:AppData/Bio/samples/sample_FASTA.csv');
   const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
   await grok.data.detectSemanticTypes(df);
@@ -202,7 +294,7 @@ export async function _testDetectorsSamplesFastaCsvPt() {
   expect(col.getTag('separator'), null);
 }
 
-export async function _testDetectorsSamplesFastaFastaPt() {
+export async function _testSamplesFastaFastaPt() {
   const fasta: string = await grok.dapi.files.readAsText('System:AppData/Bio/samples/sample_FASTA.fasta');
   const df: DG.DataFrame = importFasta(fasta)[0];
 
@@ -210,4 +302,16 @@ export async function _testDetectorsSamplesFastaFastaPt() {
   expect(col.semType, mmSemType);
   expect(col.getTag(DG.TAGS.UNITS), 'fasta:SEQ:PT');
   expect(col.getTag('separator'), null);
-}
+}
+
+export async function _testSamplesPeptidesComplexUn() {
+  const csv: string = await grok.dapi.files.readAsText('System:AppData/Bio/samples/peptides_complex_aligned.csv');
+  const df: DG.DataFrame = DG.DataFrame.fromCsv(csv);
+  await grok.data.detectSemanticTypes(df);
+
+  const col: DG.Column = df.col('AlignedSequence')!;
+  expect(col.semType, mmSemType);
+  expect(col.getTag(DG.TAGS.UNITS), 'separator:SEQ.MSA:UN');
+  expect(col.getTag('separator'), '-');
+}
+

package/src/tests/sequence-space-test.ts

@@ -1,24 +1,26 @@
 import {before, category, test, expect} from '@datagrok-libraries/utils/src/test';
-import * as DG from "datagrok-api/dg";
-import { sequenceSpace } from '../utils/sequence-space';
-import { readDataframe } from './utils';
+import * as DG from 'datagrok-api/dg';
+import {sequenceSpace} from '../utils/sequence-space';
+import {readDataframe} from './utils';
 //import * as grok from 'datagrok-api/grok';
 
 category('sequenceSpace', async () => {
+  let testFastaDf: DG.DataFrame;
 
-    let testFastaDf: DG.DataFrame;
-  
-    before(async () => {
-        //@ts-ignore
-        testFastaDf = await readDataframe('sample_FASTA.csv');
-    });
-  
-  
-    test('sequenceSpaceOpens', async () => {
-         //@ts-ignore
-        const res = await sequenceSpace(testFastaDf.col('Sequence')!, 't-SNE', 'Levenshtein', ['Embed_X', 'Embed_Y']);
-        expect(res.coordinates != undefined, true); 
-        expect(res.distance != undefined, true); 
-    });
-  
-  });
+  before(async () => {
+    testFastaDf = await readDataframe('sample_FASTA.csv');
+  });
+
+
+  test('sequenceSpaceOpens', async () => {
+    const sequenceSpaceParams = {
+      seqCol: testFastaDf.col('Sequence')!,
+      methodName: 't-SNE',
+      similarityMetric: 'Levenshtein',
+      embedAxesNames: ['Embed_X', 'Embed_Y']
+    };
+    const res = await sequenceSpace(sequenceSpaceParams);
+    expect(res.coordinates != undefined, true);
+    expect(res.distance != undefined, true);
+  });
+});

package/src/tests/utils.ts

@@ -1,8 +1,8 @@
 import * as DG from 'datagrok-api/dg';
-import * as grok from "datagrok-api/grok";
+import * as grok from 'datagrok-api/grok';
 import {expect} from '@datagrok-libraries/utils/src/test';
 import {runKalign} from '../utils/multiple-sequence-alignment';
-import { _package} from '../package-test';
+import {_package} from '../package-test';
 
 export async function loadFileAsText(name: string): Promise<string> {
   return await _package.files.readAsText(name);
@@ -15,6 +15,13 @@ export async function readDataframe(tableName: string): Promise<DG.DataFrame> {
   return df;
 }
 
+export async function createTableView(tableName: string): Promise<DG.TableView> {
+  const df = await readDataframe(tableName);
+  df.name = tableName.replace('.csv', '');
+  const view = grok.shell.addTableView(df);
+  return view;
+}
+
 
 /**
  * Tests if a table has non zero rows and columns.
@@ -35,5 +42,4 @@ export function _testTableIsNotEmpty(table: DG.DataFrame): void {
 export async function _testMSAIsCorrect(col: DG.Column): Promise<void> {
   const msaCol = await runKalign(col, true);
   expect(msaCol.toList().every((v, i) => (v == col.get(i) || v == null)), true);
-
 }

package/src/utils/convert.ts

@@ -2,23 +2,22 @@ import * as DG from 'datagrok-api/dg';
 import * as ui from 'datagrok-api/ui';
 
 export function convert(col: DG.Column): void {
-
   const current = col.tags[DG.TAGS.UNITS];
   //TODO: read all notations
   const notations = ['fasta:SEQ:NT', 'fasta:SEQ:PT', 'fasta:SEQ.MSA:NT', 'fasta:SEQ.MSA:PT', 'HELM'];
-  const choices = ui.choiceInput("convert to", "", notations.filter(e => e !== current));
+  const choices = ui.choiceInput('convert to', '', notations.filter((e) => e !== current));
 
   ui.dialog('Convert sequence')
-  .add(
+    .add(
       ui.div([
         ui.h1('current notation'),
         ui.div(current),
         choices.root
       ])
-  )
-  .onOK(() => {
-    //TODO: create new converted column
-    //col.dataFrame.columns.add();
-  })
-  .show();
+    )
+    .onOK(() => {
+      //TODO: create new converted column
+      //col.dataFrame.columns.add();
+    })
+    .show();
 }

package/src/utils/multiple-sequence-alignment.ts

@@ -56,7 +56,7 @@ export async function runKalign(col: DG.Column, isAligned = false) : Promise<DG.
 
   const aligned = _fastaToStrings(buf).slice(0, sequences.length);
   const alignedCol = DG.Column.fromStrings(`msa(${col.name})`, aligned);
-  alignedCol.setTag(DG.TAGS.UNITS, ''); 
+  alignedCol.setTag(DG.TAGS.UNITS, '');
   alignedCol.semType = C.SEM_TYPES.Macro_Molecule;
   return alignedCol;
 }

package/src/utils/sequence-activity-cliffs.ts

@@ -0,0 +1,36 @@
+import {IDrawTooltipParams} from '@datagrok-libraries/ml/src/viewers/activity-cliffs';
+import * as DG from 'datagrok-api/dg';
+import * as ui from 'datagrok-api/ui';
+import {getSimilarityFromDistance} from '@datagrok-libraries/utils/src/similarity-metrics';
+import { AvailableMetrics } from '@datagrok-libraries/ml/src/typed-metrics';
+
+export async function sequenceGetSimilarities(col: DG.Column, seq: string): Promise<DG.Column | null>{ 
+  const stringArray = col.toList();
+  const distances = new Array(stringArray.length).fill(0.0);
+  for (let i = 0; i < stringArray.length; ++i)
+    distances[i] = getSimilarityFromDistance(AvailableMetrics['String']['Levenshtein'](stringArray[i], seq));
+  return DG.Column.fromList(DG.COLUMN_TYPE.FLOAT, 'distances', distances);
+}
+
+export function drawTooltip(params: IDrawTooltipParams) {
+  params.tooltips[params.line.id] = ui.divH([]);
+  const columnNames = ui.divV([
+    ui.divText('sequense'),
+    ui.divText(params.activity.name),
+  ]);
+  columnNames.style.fontWeight = 'bold';
+  columnNames.style.display = 'flex';
+  columnNames.style.justifyContent = 'space-between';
+  params.tooltips[params.line.id].append(columnNames);
+  params.line.mols.forEach((mol: number) => {
+    const seq = ui.divText(params.df.get(params.seqCol.name, mol));
+    const activity = ui.divText(params.df.get(params.activity.name, mol).toFixed(2));
+    activity.style.display = 'flex';
+    activity.style.justifyContent = 'left';
+    activity.style.paddingLeft = '30px';
+    params.tooltips[params.line.id].append(ui.divV([
+      seq,
+      activity,
+    ], {style: {paddingLeft: '5px'}}));
+  });
+}

package/src/utils/sequence-space.ts

@@ -1,43 +1,43 @@
 import * as DG from 'datagrok-api/dg';
-import { AvailableMetrics } from '@datagrok-libraries/ml/src/typed-metrics';
+import {AvailableMetrics} from '@datagrok-libraries/ml/src/typed-metrics';
 import {reduceDimensinalityWithNormalization} from '@datagrok-libraries/ml/src/sequence-space';
 import {BitArrayMetrics, StringMetrics} from '@datagrok-libraries/ml/src/typed-metrics';
-import { Matrix } from '@datagrok-libraries/utils/src/type-declarations';
+import {Matrix} from '@datagrok-libraries/utils/src/type-declarations';
 import BitArray from '@datagrok-libraries/utils/src/bit-array';
+import {ISequenceSpaceParams} from '@datagrok-libraries/ml/src/viewers/activity-cliffs';
 
 export interface ISequenceSpaceResult {
   distance: Matrix;
   coordinates: DG.ColumnList;
 }
 
-export async function sequenceSpace(molColumn: DG.Column, methodName: string, similarityMetric: string,
-    axes: string[], options?: any): Promise<ISequenceSpaceResult> {
-    let preparedData: any;
-    if (!(molColumn!.tags[DG.TAGS.UNITS] === 'HELM')) {
-      const sep = molColumn.getTag('separator');
-      const sepFinal = sep ? sep === '.' ? '\\\.' : sep: '-';  
-      var regex = new RegExp(sepFinal, "g");
-      if (Object.keys(AvailableMetrics['String']).includes(similarityMetric)) {
-          preparedData = molColumn.toList().map((v) => v.replace(regex, '')) as string[];
-      } else {
-          preparedData = molColumn.toList().map((v) => v.replace(regex, '')) as string[];
-      }
-    } else {
-      preparedData = molColumn.toList();
-    }
-    
-    const sequenceSpaceResult = await reduceDimensinalityWithNormalization(
-      preparedData,
-      methodName,
-      similarityMetric as StringMetrics|BitArrayMetrics,
-      options);
-    const cols: DG.Column[] = axes.map((name, index) => DG.Column.fromFloat32Array(name, sequenceSpaceResult.embedding[index]))
-    return {distance: sequenceSpaceResult.distance, coordinates: new DG.ColumnList(cols)};
+export async function sequenceSpace(spaceParams: ISequenceSpaceParams): Promise<ISequenceSpaceResult> {
+  let preparedData: any;
+  if (!(spaceParams.seqCol!.tags[DG.TAGS.UNITS] === 'HELM')) {
+    const sep = spaceParams.seqCol.getTag('separator');
+    const sepFinal = sep ? sep === '.' ? '\\\.' : sep : '-';
+    const regex = new RegExp(sepFinal, 'g');
+    if (Object.keys(AvailableMetrics['String']).includes(spaceParams.similarityMetric))
+      preparedData = spaceParams.seqCol.toList().map((v) => v.replace(regex, '')) as string[];
+    else
+      preparedData = spaceParams.seqCol.toList().map((v) => v.replace(regex, '')) as string[];
+  } else {
+    preparedData = spaceParams.seqCol.toList();
   }
 
+  const sequenceSpaceResult = await reduceDimensinalityWithNormalization(
+    preparedData,
+    spaceParams.methodName,
+    spaceParams.similarityMetric as StringMetrics | BitArrayMetrics,
+    spaceParams.options);
+  const cols: DG.Column[] = spaceParams.embedAxesNames.map(
+    (name, index) => DG.Column.fromFloat32Array(name, sequenceSpaceResult.embedding[index]));
+  return {distance: sequenceSpaceResult.distance, coordinates: new DG.ColumnList(cols)};
+}
+
 
-export function getEmbeddingColsNames(df: DG.DataFrame){
-    const axes = ['Embed_X', 'Embed_Y'];
-    const colNameInd = df.columns.names().filter((it) => it.includes(axes[0])).length + 1;
-    return axes.map((it) => `${it}_${colNameInd}`);
-  }
+export function getEmbeddingColsNames(df: DG.DataFrame) {
+  const axes = ['Embed_X', 'Embed_Y'];
+  const colNameInd = df.columns.names().filter((it) => it.includes(axes[0])).length + 1;
+  return axes.map((it) => `${it}_${colNameInd}`);
+}