@datagrok/bio 2.4.13 → 2.4.14

package/package.json

@@ -5,7 +5,7 @@
     "name": "Leonid Stolbov",
     "email": "lstolbov@datagrok.ai"
   },
-  "version": "2.4.13",
+  "version": "2.4.14",
   "description": "Bioinformatics support (import/export of sequences, conversion, visualization, analysis). [See more](https://github.com/datagrok-ai/public/blob/master/packages/Bio/README.md) for details.",
   "repository": {
     "type": "git",

package/src/package.ts

@@ -409,8 +409,8 @@ export async function toAtomicLevel(df: DG.DataFrame, macroMolecule: DG.Column):
 //top-menu: Bio | Alignment | MSA...
 //name: MSA...
 //tags: bio, panel
-export function multipleSequenceAlignmentAny(col: DG.Column<string> | null = null): void {
-  multipleSequenceAlignmentUI(col);
+export function multipleSequenceAlignmentAny(): void {
+  multipleSequenceAlignmentUI();
 }
 
 //top-menu: Bio | Structure | Composition Analysis

package/src/tests/msa-tests.ts

@@ -3,8 +3,9 @@ import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
 
 import {category, expect, expectArray, test} from '@datagrok-libraries/utils/src/test';
-
+import {ALIGNMENT, ALPHABET, NOTATION, TAGS as bioTAGS} from '@datagrok-libraries/bio/src/utils/macromolecule';
 import {runKalign} from '../utils/multiple-sequence-alignment';
+import {multipleSequenceAlignmentUI} from '../utils/multiple-sequence-alignment-ui';
 //import * as grok from 'datagrok-api/grok';
 
 export const _package = new DG.Package();
@@ -31,9 +32,46 @@ FWR-WYVKHPFWR-WYVKHPFWR-WYVKHPFWR-WYVKHPFWR-WYVKHPFWR-WYVKHPFWR-WYVKHPFWR-WYVKHP
 YNR-WYVKHPYNR-WYVKHPYNR-WYVKHPYNR-WYVKHPYNR-WYVKHPYNR-WYVKHPYNR-WYVKHPYNR-WYVKHP
 MWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHP`;
 
-  // test('test_table.is_not_empty', async () => {
-  //   await _testTableIsNotEmpty(table);
-  // });
+
+  const helmFromCsv = `seq
+  PEPTIDE1{meI.hHis.Aca.N.T.dE.Thr_PO3H2}$$$$
+  PEPTIDE1{meI.Aca.N.T.dE.Thr_PO3H2}$$$$
+  PEPTIDE1{hHis.Aca.N.T.dE.Thr_PO3H2}$$$$`;
+
+  const helmToCsv = `seq
+  meI.hHis.Aca.N.T.dE.Thr_PO3H2
+  .meI.Aca.N.T.dE.Thr_PO3H2
+  .hHis.Aca.N.T.dE.Thr_PO3H2`;
+
+  const longHelmFromCsv = `seq
+  PEPTIDE1{meI.hHis.Aca.N.T.dE.Thr_PO3H2.Aca.D-Tyr_Et.dV.E.N.D-Orn.D-aThr.Phe_4Me.Thr_PO3H2}$$$$
+  PEPTIDE1{meI.hHis.Aca.Cys_SEt.T.dK.Tyr_PO3H2.D-Chg.dV.Phe_ab-dehydro.N.D-Orn.D-aThr.Phe_4Me}$$$$
+  PEPTIDE1{Lys_Boc.hHis.Aca.Cys_SEt.T.dK.Tyr_PO3H2.D-Chg.dV.Thr_PO3H2.N.D-Orn.D-aThr.Phe_4Me}$$$$`;
+
+  const longHelmToCsv = `seq
+  meI.hHis.Aca.N.T.dE.Thr_PO3H2.Aca.D-Tyr_Et.dV.E.N.D-Orn.D-aThr.Phe_4Me.Thr_PO3H2
+  meI.hHis.Aca.Cys_SEt.T.dK..Tyr_PO3H2.D-Chg.dV.Phe_ab-dehydro.N.D-Orn.D-aThr.Phe_4Me.
+  Lys_Boc.hHis.Aca.Cys_SEt.T.dK..Tyr_PO3H2.D-Chg.dV.Thr_PO3H2.N.D-Orn.D-aThr.Phe_4Me.`;
+
+  const SeparatorFromCsv = `seq
+  F-W-P-H-E-Y
+  Y-N-R-Q-W-Y-V
+  M-K-P-S-E-Y-V`;
+
+  const SeparatorToCsv = `seq
+  FWPHEY-
+  YNRQWYV
+  MKPSEYV`;
+
+  const SeparatorLongFromCsv = `seq
+  M-I-E-V-F-L-F-G-I-V-L-G-L-I-P-I-T-L-A-G-L-F-V-T-A-Y-L-Q-Y-R-R-G-D-Q-L-D-L
+  M-M-E-L-V-L-K-T-I-I-G-P-I-V-V-G-V-V-L-R-I-V-D-K-W-L-N-K-D-K
+  M-D-R-T-D-E-V-S-N-H-T-H-D-K-P-T-L-T-W-F-E-E-I-F-E-E-Y-H-S-P-F-H-N`;
+
+  const SeparatorLongToCsv = `seq
+  MIEV-FLFGIVLGLIPITLAGLFVTAYLQYRRGDQLDL
+  MMEL-VLKTII-GPIVVGVVLRIVDKWLNKDK------
+  MDRTDEVSNHTHDKPTLTWFEEIFEEYHSPFHN-----`;
 
   test('isCorrect', async () => {
     await _testMsaIsCorrect(fromCsv, toCsv);
@@ -42,6 +80,26 @@ MWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHPMWRSWYCKHP
   test('isCorrectLong', async () => {
     await _testMsaIsCorrect(longFromCsv, longToCsv);
   });
+
+  test('isCorrectHelm', async () => {
+    await _testMSAOnColumn(helmFromCsv, helmToCsv, NOTATION.HELM, NOTATION.SEPARATOR, undefined, 'mafft');
+  }, {skipReason: 'GROK-13053'});
+
+  test('isCorrectHelmLong', async () => {
+    await _testMSAOnColumn(longHelmFromCsv, longHelmToCsv, NOTATION.HELM, NOTATION.SEPARATOR, undefined, 'mafft');
+  }, {skipReason: 'GROK-13053'});
+
+  test('isCorrectSeparator', async () => {
+    await _testMSAOnColumn(
+      SeparatorFromCsv, SeparatorToCsv, NOTATION.SEPARATOR, NOTATION.FASTA, ALPHABET.PT
+    );
+  });
+
+  test('isCorrectSeparatorLong', async () => {
+    await _testMSAOnColumn(
+      SeparatorLongFromCsv, SeparatorLongToCsv, NOTATION.SEPARATOR, NOTATION.FASTA, ALPHABET.PT
+    );
+  });
 });
 
 async function _testMsaIsCorrect(srcCsv: string, tgtCsv: string): Promise<void> {
@@ -58,3 +116,33 @@ async function _testMsaIsCorrect(srcCsv: string, tgtCsv: string): Promise<void>
   const msaCol: DG.Column = await runKalign(srcCol, true);
   expectArray(msaCol.toList(), tgtCol.toList());
 }
+
+async function _testMSAOnColumn(
+  srcCsv: string, tgtCsv: string,
+  srcNotation: NOTATION, tgtNotation: NOTATION, alphabet?: ALPHABET, pepseaMethod?: string
+): Promise<void> {
+  const srcDf: DG.DataFrame = DG.DataFrame.fromCsv(srcCsv);
+  const tgtDf: DG.DataFrame = DG.DataFrame.fromCsv(tgtCsv);
+
+  const srcSeqCol = srcDf.getCol('seq')!;
+  const tgtCol = tgtDf.getCol('seq')!;
+  const srcCol: DG.Column = srcDf.getCol('seq')!;
+  const semType: string = await grok.functions
+    .call('Bio:detectMacromolecule', {col: srcCol}) as unknown as string;
+  if (semType)
+    srcCol.semType = semType;
+
+  await grok.data.detectSemanticTypes(srcDf);
+  expect(srcSeqCol.semType, DG.SEMTYPE.MACROMOLECULE);
+  expect(srcSeqCol.getTag(DG.TAGS.UNITS), srcNotation);
+  if (alphabet)
+    expect(srcSeqCol.getTag(bioTAGS.alphabet), alphabet);
+
+  const msaSeqCol = await multipleSequenceAlignmentUI(srcSeqCol, pepseaMethod);
+  expect(msaSeqCol.semType, DG.SEMTYPE.MACROMOLECULE);
+  expect(msaSeqCol.getTag(DG.TAGS.UNITS), tgtNotation);
+  expect(msaSeqCol.getTag(bioTAGS.aligned), ALIGNMENT.SEQ_MSA);
+  if (alphabet)
+    expect(msaSeqCol.getTag(bioTAGS.alphabet), alphabet);
+  expectArray(msaSeqCol.toList(), tgtCol.toList());
+}

package/src/tests/renderers-test.ts

@@ -10,6 +10,7 @@ import {generateLongSequence, generateManySequences, performanceTest} from './ut
 import {errorToConsole} from '@datagrok-libraries/utils/src/to-console';
 import {ALIGNMENT, ALPHABET, NOTATION, TAGS as bioTAGS} from '@datagrok-libraries/bio/src/utils/macromolecule';
 import {UnitsHandler} from '@datagrok-libraries/bio/src/utils/units-handler';
+import {multipleSequenceAlignmentUI} from '../utils/multiple-sequence-alignment-ui';
 
 category('renderers', () => {
   let tvList: DG.TableView[];
@@ -145,17 +146,17 @@ category('renderers', () => {
     expect(srcSeqCol.getTag(bioTAGS.alphabet), ALPHABET.PT);
     expect(srcSeqCol.getTag(DG.TAGS.CELL_RENDERER), 'sequence');
 
-    const msaSeqCol = multipleSequenceAlignmentAny(srcSeqCol);
+    const msaSeqCol = await multipleSequenceAlignmentUI(srcSeqCol);
     tv.grid.invalidate();
 
-    // expect(msaSeqCol.semType, DG.SEMTYPE.MACROMOLECULE);
-    // expect(msaSeqCol.getTag(DG.TAGS.UNITS), NOTATION.FASTA);
-    // expect(msaSeqCol.getTag(bioTAGS.aligned), ALIGNMENT.SEQ_MSA);
-    // expect(msaSeqCol.getTag(bioTAGS.alphabet), ALPHABET.PT);
-    // expect(msaSeqCol.getTag(DG.TAGS.CELL_RENDERER), 'sequence');
+    expect(msaSeqCol.semType, DG.SEMTYPE.MACROMOLECULE);
+    expect(msaSeqCol.getTag(DG.TAGS.UNITS), NOTATION.FASTA);
+    expect(msaSeqCol.getTag(bioTAGS.aligned), ALIGNMENT.SEQ_MSA);
+    expect(msaSeqCol.getTag(bioTAGS.alphabet), ALPHABET.PT);
+    expect(msaSeqCol.getTag(DG.TAGS.CELL_RENDERER), 'sequence');
 
     // check newColumn with UnitsHandler constructor
-    // const uh: UnitsHandler = new UnitsHandler(msaSeqCol);
+    const uh: UnitsHandler = new UnitsHandler(msaSeqCol);
 
     dfList.push(df);
     tvList.push(tv);

package/src/utils/multiple-sequence-alignment-ui.ts

@@ -6,76 +6,144 @@ import {ALPHABET, NOTATION} from '@datagrok-libraries/bio/src/utils/macromolecul
 import {runKalign} from './multiple-sequence-alignment';
 import {pepseaMethods, runPepsea} from './pepsea';
 import {checkInputColumnUI} from './check-input-column';
+import {NotationConverter} from '@datagrok-libraries/bio/src/utils/notation-converter';
+import {_package} from '../package';
 
-export function multipleSequenceAlignmentUI(col: DG.Column<string> | null = null) {
-  const table = col?.dataFrame ?? grok.shell.t;
-  const seqCol = col ?? table.columns.bySemType(DG.SEMTYPE.MACROMOLECULE);
-  if (seqCol == null) {
-    grok.shell.warning(`MSAError: dataset doesn't conain any Macromolecule column`);
-    return;
+export class MsaWarning extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
   }
+}
+
+export async function multipleSequenceAlignmentUI(
+  col: DG.Column<string> | null = null,
+  pepseaMethod: typeof pepseaMethods[number] = pepseaMethods[0]
+): Promise<DG.Column> {
+  return new Promise(async (resolve, reject) => {
+    const table = col?.dataFrame ?? grok.shell.t;
+    const seqCol = col ?? table.columns.bySemType(DG.SEMTYPE.MACROMOLECULE);
+    if (seqCol == null) {
+      const errMsg = `MSAError: dataset doesn't conain any Macromolecule column`;
+      grok.shell.warning(errMsg);
+      reject(new MsaWarning(errMsg));
+    }
 
-  let performAlignment: () => Promise<DG.Column<string> | null> = async () => null;
-  const methodInput = ui.choiceInput('Method', pepseaMethods[0], pepseaMethods);
-  methodInput.setTooltip('Alignment method');
-  const gapOpenInput = ui.floatInput('Gap open', 1.53);
-  gapOpenInput.setTooltip('Gap opening penalty at group-to-group alignment');
-  const gapExtendInput = ui.floatInput('Gap extend', 0);
-  gapExtendInput.setTooltip('Gap extension penalty to skip the alignment');
-  const inputRootStyles = [methodInput.root.style, gapOpenInput.root.style, gapExtendInput.root.style];
+    // UI
+    const methodInput = ui.choiceInput('Method', pepseaMethod, pepseaMethods);
+    methodInput.setTooltip('Alignment method');
+    const gapOpenInput = ui.floatInput('Gap open', 1.53);
+    gapOpenInput.setTooltip('Gap opening penalty at group-to-group alignment');
+    const gapExtendInput = ui.floatInput('Gap extend', 0);
+    gapExtendInput.setTooltip('Gap extension penalty to skip the alignment');
+    const inputRootStyles = [methodInput.root.style, gapOpenInput.root.style, gapExtendInput.root.style];
+    let performAlignment: (() => Promise<DG.Column<string>>) | undefined;
 
-  const colInput = ui.columnInput('Sequence', table, seqCol, () => {
-    const potentialCol = colInput.value;
-    const unusedName = table.columns.getUnusedName(`msa(${potentialCol.name})`);
+    // TODO: allow only macromolecule colums to be chosen
+    const colInput = ui.columnInput('Sequence', table, seqCol, () => {
+      performAlignment = onColInputChange(
+        colInput.value, table, inputRootStyles, methodInput, clustersColInput, gapOpenInput, gapExtendInput
+      );
+    }
+    ) as DG.InputBase<DG.Column<string>>;
+    colInput.setTooltip('Sequences column to use for alignment');
+    const clustersColInput = ui.columnInput('Clusters', table, null);
+    clustersColInput.nullable = true;
+    colInput.fireChanged();
+    //if column is specified (from tests), run alignment and resolve with the result
+    if (col) {
+      performAlignment = onColInputChange(
+        col, table, inputRootStyles, methodInput, clustersColInput, gapOpenInput, gapExtendInput
+      );
 
-    if (checkInputColumnUI(
-      potentialCol, potentialCol.name, [NOTATION.FASTA], [ALPHABET.DNA, ALPHABET.RNA, ALPHABET.PT], false)) {
+      await onDialogOk(colInput, table, performAlignment, resolve, reject);
+      return;
+    }
+    const dlg = ui.dialog('MSA')
+      .add(colInput)
+      .add(clustersColInput)
+      .add(methodInput)
+      .add(gapOpenInput)
+      .add(gapExtendInput)
+      .onOK(async () => {
+        await onDialogOk(colInput, table, performAlignment, resolve, reject);
+      })
+      .show();
+  });
+}
+
+async function onDialogOk(
+  colInput: DG.InputBase< DG.Column<any>>,
+  table: DG.DataFrame,
+  performAlignment: (() => Promise<DG.Column<string>>) | undefined,
+  resolve: (value: DG.Column<any>) => void,
+  reject: (reason: any) => void
+): Promise<void> {
+  let msaCol: DG.Column<string> | null = null;
+  const pi = DG.TaskBarProgressIndicator.create('Analyze for MSA ...');
+  try {
+    colInput.fireChanged();
+    if (colInput.value.semType !== DG.SEMTYPE.MACROMOLECULE)
+      throw new Error('Chosen column has to be of Macromolecule semantic type');
+    if (performAlignment === undefined) // value can only be undefined when column can't be processed with either method
+      throw new Error('Invalid column format');
+    msaCol = await performAlignment(); // progress
+    if (msaCol == null)
+      return grok.shell.warning('Wrong column format');
+
+    table.columns.add(msaCol);
+    await grok.data.detectSemanticTypes(table);
+
+    resolve(msaCol);
+  } catch (err: any) {
+    const errMsg: string = err instanceof Error ? err.message : err.toString();
+    grok.shell.error(errMsg);
+    reject(err);
+  } finally {
+    pi.close();
+  }
+}
+
+
+function onColInputChange(
+  col: DG.Column<string>,
+  table: DG.DataFrame,
+  inputRootStyles: CSSStyleDeclaration[],
+  methodInput: DG.InputBase<string | null>,
+  clustersColInput: DG.InputBase<DG.Column<any> | null>,
+  gapOpenInput: DG.InputBase<number | null>,
+  gapExtendInput: DG.InputBase<number | null>
+): (() => Promise<DG.Column<string>>) | undefined {
+  try {
+    if (col.semType !== DG.SEMTYPE.MACROMOLECULE)
+      return;
+    const unusedName = table.columns.getUnusedName(`msa(${col.name})`);
+
+    if (checkInputColumnUI(col, col.name,
+      [NOTATION.FASTA, NOTATION.SEPARATOR], [ALPHABET.DNA, ALPHABET.RNA, ALPHABET.PT], false)
+    ) { // Kalign - natural alphabets. if the notation is separator, convert to fasta and then run kalign
       for (const inputRootStyle of inputRootStyles)
         inputRootStyle.display = 'none';
-
-      performAlignment = () => runKalign(potentialCol, false, unusedName, clustersColInput.value);
-    } else if (checkInputColumnUI(potentialCol, potentialCol.name, [NOTATION.HELM], [], false)) {
+      const potentialColNC = new NotationConverter(col);
+      const performCol: DG.Column<string> = potentialColNC.isFasta() ? col :
+        potentialColNC.convert(NOTATION.FASTA);
+      return async () => await runKalign(performCol, false, unusedName, clustersColInput.value);
+    } else if (checkInputColumnUI(col, col.name,
+      [NOTATION.HELM], [], false)
+    ) { // PepSeA branch - Helm notation or separator notation with unknown alphabets
       for (const inputRootStyle of inputRootStyles)
-        inputRootStyle.display = 'initial';
+        inputRootStyle.removeProperty('display');
 
-      performAlignment = () => runPepsea(potentialCol, unusedName, methodInput.value!, gapOpenInput.value!,
-        gapExtendInput.value!, clustersColInput.value);
+      return async () => await runPepsea(col, unusedName, methodInput.value!,
+          gapOpenInput.value!, gapExtendInput.value!, clustersColInput.value);
     } else {
       for (const inputRootStyle of inputRootStyles)
         inputRootStyle.display = 'none';
 
-      performAlignment = async () => null;
+      return;
     }
-  }) as DG.InputBase<DG.Column<string>>;
-  colInput.setTooltip('Sequences column to use for alignment');
-  colInput.fireChanged();
-
-  const clustersColInput = ui.columnInput('Clusters', table, null);
-  clustersColInput.nullable = true;
-
-  let msaCol: DG.Column<string> | null = null;
-  ui.dialog('MSA')
-    .add(colInput)
-    .add(clustersColInput)
-    .add(methodInput)
-    .add(gapOpenInput)
-    .add(gapExtendInput)
-    .onOK(async () => {
-      const pi = DG.TaskBarProgressIndicator.create('Analyze for MSA ...');
-      try {
-        colInput.fireChanged();
-        msaCol = await performAlignment(); // progress
-        if (msaCol == null)
-          return grok.shell.warning('Wrong column format');
-
-        table.columns.add(msaCol);
-        await grok.data.detectSemanticTypes(table);
-      } catch (err: any) {
-        const errMsg: string = err instanceof Error ? err.message : err.toString();
-        grok.shell.error(errMsg);
-      } finally {
-        pi.close();
-      }
-    })
-    .show();
+  } catch (err: any) {
+    const errMsg: string = err instanceof Error ? err.message : err.toString();
+    grok.shell.error(errMsg);
+    _package.logger.error(errMsg);
+  }
 }

package/src/utils/multiple-sequence-alignment.ts

@@ -2,7 +2,7 @@
 import * as DG from 'datagrok-api/dg';
 
 import {FastaFileHandler} from '@datagrok-libraries/bio/src/utils/fasta-handler';
-import {TAGS as bioTAGS} from '@datagrok-libraries/bio/src/utils/macromolecule';
+import {ALIGNMENT, TAGS as bioTAGS} from '@datagrok-libraries/bio/src/utils/macromolecule';
 //@ts-ignore: there are no types for this library
 import Aioli from '@biowasm/aioli';
 
@@ -25,7 +25,8 @@ function _stringsToFasta(sequences: string[]): string {
  *
  * @param {DG.Column} srcCol Column with sequences.
  * @param {boolean} isAligned Whether the column is aligned.
- * @param {string} unUsedName
+ * @param {string | undefined} unUsedName
+ * @param {DG.Column | null} clustersCol Column with clusters.
  * @return {Promise<DG.Column>} Aligned sequences.
  */
 export async function runKalign(srcCol: DG.Column<string>, isAligned: boolean = false, unUsedName: string = '',
@@ -40,7 +41,7 @@ export async function runKalign(srcCol: DG.Column<string>, isAligned: boolean =
   if (clustersCol.type != DG.COLUMN_TYPE.STRING)
     clustersCol = clustersCol.convertTo(DG.TYPE.STRING);
   clustersCol.compact();
-  
+
   //TODO: use fixed-size inner arrays, but first need to expose the method to get each category count
   const clustersColCategories = clustersCol.categories;
   const clustersColData = clustersCol.getRawData();
@@ -61,8 +62,6 @@ export async function runKalign(srcCol: DG.Column<string>, isAligned: boolean =
   for (let clusterIdx = 0; clusterIdx < clustersColCategories.length; ++clusterIdx) {
     const clusterSequences = fastaSequences[clusterIdx];
     const fasta = _stringsToFasta(clusterSequences);
-    
-    console.log(['fasta.length =', fasta.length]);
 
     await CLI.fs.writeFile(fastaInputFilename, fasta);
     const output = await CLI.exec(`kalign ${fastaInputFilename} -f fasta -o ${fastaOutputFilename}`);
@@ -82,8 +81,7 @@ export async function runKalign(srcCol: DG.Column<string>, isAligned: boolean =
   // units
   const srcUnits = srcCol.getTag(DG.TAGS.UNITS);
   //aligned
-  const srcAligned = srcCol.getTag(bioTAGS.aligned);
-  const tgtAligned = srcAligned + '.MSA';
+  const tgtAligned = ALIGNMENT.SEQ_MSA;
   //alphabet
   const srcAlphabet = srcCol.getTag(bioTAGS.alphabet);