@datagrok/bio 1.4.0 → 1.4.1

package/package.json

@@ -2,7 +2,7 @@
   "name": "@datagrok/bio",
   "beta": false,
   "friendlyName": "Bio",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "Bio is a [package](https://datagrok.ai/help/develop/develop#packages) for the [Datagrok](https://datagrok.ai) platform",
   "repository": {
     "type": "git",
@@ -13,6 +13,7 @@
     "@biowasm/aioli": ">=2.4.0",
     "@datagrok-libraries/bio": "^2.2.0",
     "@datagrok-libraries/utils": "^0.4.2",
+    "@datagrok-libraries/ml": "^2.0.2",
     "cash-dom": "latest",
     "datagrok-api": "^1.4.11",
     "dayjs": "latest",
@@ -35,7 +36,8 @@
   "scripts": {
     "link-api": "npm link datagrok-api",
     "link-bio": "npm link @datagrok-libraries/bio",
-    "link-all": "npm link datagrok-api @datagrok-libraries/bio",
+    "link-ml": "npm link @datagrok-libraries/ml",
+    "link-all": "npm link datagrok-api @datagrok-libraries/utils @datagrok-libraries/bio @datagrok-libraries/ml",
     "debug-sequences1": "grok publish --rebuild",
     "release-sequences1": "grok publish --rebuild --release",
     "build-sequences1": "webpack",
@@ -48,7 +50,8 @@
     "lint": "eslint \"./src/**/*.ts\"",
     "lint-fix": "eslint \"./src/**/*.ts\" --fix",
     "test": "jest",
-    "test-local": "set HOST=localhost && jest"
+    "test-local": "set HOST=localhost && jest",
+    "build-bio-local": "npm --prefix ./../../js-api run build && npm --prefix ./../../libraries/utils run build && npm --prefix ./../../libraries/ml run build && npm run build && npm --prefix ./../../libraries/bio run build && npm run build"
   },
   "canEdit": [
     "Developers"

package/src/package-test.ts

@@ -6,8 +6,9 @@ import './tests/WebLogo-test';
 import './tests/Palettes-test';
 import './tests/detectors-test';
 import './tests/msa-tests';
+import './tests/sequence-space-test';
 
-export const _packageTest = new DG.Package();
+export const _package = new DG.Package();
 export {tests};
 
 /** For the 'test' function argument names are fixed as 'category' and 'test' because of way it is called. */

package/src/package.ts

@@ -2,7 +2,6 @@
 import * as grok from 'datagrok-api/grok';
 import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
-
 import {SequenceAlignment, Aligned} from './seq_align';
 
 export const _package = new DG.Package();
@@ -10,10 +9,10 @@ export const _package = new DG.Package();
 import {WebLogo} from '@datagrok-libraries/bio/src/viewers/web-logo';
 import {VdRegionsViewer} from './viewers/vd-regions-viewer';
 import {runKalign, testMSAEnoughMemory} from './utils/multiple-sequence-alignment';
+import {convert} from './utils/convert';
 import {TableView} from 'datagrok-api/dg';
-import {mmSemType} from './const';
-import {Nucleotides} from '@datagrok-libraries/bio/src/nucleotides';
-import {Aminoacids} from '@datagrok-libraries/bio/src/aminoacids';
+import { getEmbeddingColsNames, sequenceSpace } from './utils/sequence-space';
+import { AvailableMetrics } from '@datagrok-libraries/ml/src/typed-metrics';
 
 //name: sequenceAlignment
 //input: string alignType {choices: ['Local alignment', 'Global alignment']}
@@ -60,12 +59,23 @@ export async function activityCliffs(df: DG.DataFrame, smiles: DG.Column, activi
 //top-menu: Bio | Sequence Space...
 //name: Sequence Space
 //input: dataframe table
-//input: column smiles { semType: Macromolecule }
-//input: string methodName { choices:["UMAP", "t-SNE", "SPE", "pSPE", "OriginalSPE"] }
-//input: string similarityMetric { choices:["Tanimoto", "Asymmetric", "Cosine", "Sokal"] }
+//input: column macroMolecule { semType: Macromolecule }
+//input: string methodName { choices:["UMAP", "t-SNE", "SPE"] }
+//input: string similarityMetric { choices:["Levenshtein", "Tanimoto"] }
 //input: bool plotEmbeddings = true
-export async function chemSpaceTopMenu(table: DG.DataFrame, smiles: DG.Column, methodName: string,
-  similarityMetric: string = 'Tanimoto', plotEmbeddings: boolean): Promise<void> {
+export async function sequenceSpaceTopMenu(table: DG.DataFrame, macroMolecule: DG.Column, methodName: string,
+  similarityMetric: string = 'Levenshtein', plotEmbeddings: boolean) : Promise<void> {
+    const embedColsNames = getEmbeddingColsNames(table);
+    const sequenceSpaceRes = await sequenceSpace(macroMolecule, methodName, similarityMetric, embedColsNames);
+    const embeddings = sequenceSpaceRes.coordinates;
+    for (const col of embeddings)
+      table.columns.add(col);
+    if (plotEmbeddings) {
+      for (let v of grok.shell.views) {
+        if (v.name === table.name)
+          (v as DG.TableView).scatterPlot({x: embedColsNames[0], y: embedColsNames[1]});
+      }
+    } 
 };
 
 //top-menu: Bio | MSA...
@@ -97,44 +107,50 @@ export async function compositionAnalysis(): Promise<void> {
   }
 }
 
+// helper function for importFasta
+function parseMacromolecule(
+  fileContent: string,
+  startOfSequence: number,
+  endOfSequence: number
+): string {
+  const seq = fileContent.slice(startOfSequence, endOfSequence);
+  const seqArray = seq.split(/\s/);
+  return seqArray.join('');
+}
+
 //name: importFasta
 //description: Opens FASTA file
 //tags: file-handler
 //meta.ext: fasta, fna, ffn, faa, frn, fa
 //input: string content
 //output: list tables
-export function importFasta(content: string): DG.DataFrame [] {
-  const regex = /^>(.*)$/gm;
-  const descriptions = [];
-  const sequences = [];
-  let index = 0;
-  let match;
-  while (match = regex.exec(content)) {
-    descriptions.push(content.substring(match.index + 1, regex.lastIndex));
-    if (index !== 0)
-      sequences.push(content.substring(index, regex.lastIndex));
-    index = regex.lastIndex + 1;
+export function importFasta(fileContent: string): DG.DataFrame [] {
+  const regex = /^>(.*)$/gm; // match the line starting with >
+  const descriptionsArray = [];
+  const sequencesArray: string[] = [];
+  let startOfSequence = 0;
+  let match; // match.index is the beginning of the matched line
+  while (match = regex.exec(fileContent)) {
+    const description = fileContent.substring(match.index + 1, regex.lastIndex);
+    descriptionsArray.push(description);
+    if (startOfSequence !== 0)
+      sequencesArray.push(parseMacromolecule(fileContent, startOfSequence, match.index));
+    startOfSequence = regex.lastIndex + 1;
   }
-  sequences.push(content.substring(index));
-  const descriptionsCol = DG.Column.fromStrings('description', descriptions);
-  const sequenceCol = DG.Column.fromStrings('sequence', sequences);
-
-  const stats: { freq: { [m: string]: number }, sameLength: boolean } = WebLogo.getStats(sequenceCol, 5, WebLogo.splitterAsFasta);
-  const seqType = stats.sameLength ? 'SEQ.MSA' : 'SEQ';
-  const alphabetCandidates: [string, Set<string>][] = [
-    ['NT', new Set(Object.keys(Nucleotides.Names)),],
-    ['PT', new Set(Object.keys(Aminoacids.Names)),],
-  ];
-  // Calculate likelihoods for alphabet_candidates
-  const alphabetCandidatesSim: number[] = alphabetCandidates.map(
-    (c) => WebLogo.getAlphabetSimilarity(stats.freq, c[1]));
-  const maxCos = Math.max(...alphabetCandidatesSim);
-  const alphabet = maxCos > 0.65 ? alphabetCandidates[alphabetCandidatesSim.indexOf(maxCos)][0] : 'UN';
-  sequenceCol.semType = mmSemType;
-  sequenceCol.setTag(DG.TAGS.UNITS, `fasta:${seqType}:${alphabet}`);
-
+  sequencesArray.push(parseMacromolecule(fileContent, startOfSequence, -1));
+  const descriptionsArrayCol = DG.Column.fromStrings('description', descriptionsArray);
+  const sequenceCol = DG.Column.fromStrings('sequence', sequencesArray);
+  sequenceCol.semType = 'Macromolecule';
   return [DG.DataFrame.fromColumns([
-    descriptionsCol,
+    descriptionsArrayCol,
     sequenceCol,
   ])];
 }
+
+//name: Bio | Convert
+//friendly-name: Bio | Convert
+//tags: panel, bio
+//input: column col {semType: Macromolecule}
+export function convertPanel(col: DG.Column): void {
+  convert(col);
+}

package/src/tests/sequence-space-test.ts

@@ -0,0 +1,24 @@
+import {before, category, test, expect} from '@datagrok-libraries/utils/src/test';
+import * as DG from "datagrok-api/dg";
+import { sequenceSpace } from '../utils/sequence-space';
+import { readDataframe } from './utils';
+//import * as grok from 'datagrok-api/grok';
+
+category('sequenceSpace', async () => {
+
+    let testFastaDf: DG.DataFrame;
+  
+    before(async () => {
+        //@ts-ignore
+        testFastaDf = await readDataframe('sample_FASTA.csv');
+    });
+  
+  
+    test('sequenceSpaceOpens', async () => {
+         //@ts-ignore
+        const res = await sequenceSpace(testFastaDf.col('Sequence')!, 't-SNE', 'Levenshtein', ['Embed_X', 'Embed_Y']);
+        expect(res.coordinates != undefined, true); 
+        expect(res.distance != undefined, true); 
+    });
+  
+  });

package/src/tests/utils.ts

@@ -1,7 +1,20 @@
 import * as DG from 'datagrok-api/dg';
-
+import * as grok from "datagrok-api/grok";
 import {expect} from '@datagrok-libraries/utils/src/test';
 import {runKalign} from '../utils/multiple-sequence-alignment';
+import { _package} from '../package-test';
+
+export async function loadFileAsText(name: string): Promise<string> {
+  return await _package.files.readAsText(name);
+}
+
+export async function readDataframe(tableName: string): Promise<DG.DataFrame> {
+  const file = await loadFileAsText(tableName);
+  const df = DG.DataFrame.fromCsv(file);
+  df.name = tableName.replace('.csv', '');
+  return df;
+}
+
 
 /**
  * Tests if a table has non zero rows and columns.

package/src/utils/convert.ts

@@ -0,0 +1,24 @@
+import * as DG from 'datagrok-api/dg';
+import * as ui from 'datagrok-api/ui';
+
+export function convert(col: DG.Column): void {
+
+  const current = col.tags[DG.TAGS.UNITS];
+  //TODO: read all notations
+  const notations = ['fasta:SEQ:NT', 'fasta:SEQ:PT', 'fasta:SEQ.MSA:NT', 'fasta:SEQ.MSA:PT', 'HELM'];
+  const choices = ui.choiceInput("convert to", "", notations.filter(e => e !== current));
+
+  ui.dialog('Convert sequence')
+  .add(
+      ui.div([
+        ui.h1('current notation'),
+        ui.div(current),
+        choices.root
+      ])
+  )
+  .onOK(() => {
+    //TODO: create new converted column
+    //col.dataFrame.columns.add();
+  })
+  .show();
+}

package/src/utils/multiple-sequence-alignment.ts

@@ -26,37 +26,6 @@ function _fastaToStrings(fasta: string): string[] {
   return fasta.replace(/>sample\d+(\r\n|\r|\n)/g, '').split('\n');
 }
 
-/**
- * Converts aligned sequence to semantic type format.
- *
- * @param {string} seq Source sequence.
- * @return {string} Formatted sequence.
- */
-function _castAligned(seq: string): string {
-  let delimited = '';
-
-  for (const char of seq)
-    delimited += char == '-' ? char : `-${char}`;
-
-  return delimited;
-}
-
-/**
- * Formats a batch of sequences to correspond the semantic type.
- *
- * @param {string[]} alignment List of aligned sequences.
- * @return {string[]} Formatted sequences.
- */
-function _stringsToAligned(alignment: string[]): string[] {
-  const nItems = alignment.length;
-  const aligned = new Array<string>(nItems);
-
-  for (let i = 0; i < nItems; ++i)
-    aligned[i] = _castAligned(alignment[i]);
-
-  return aligned;
-}
-
 /**
  * Runs Aioli environment with kalign tool.
  *
@@ -86,7 +55,7 @@ export async function runKalign(col: DG.Column, isAligned = false) : Promise<DG.
   console.warn(output);
 
   const aligned = _fastaToStrings(buf).slice(0, sequences.length);
-  const alignedCol = DG.Column.fromStrings(`msa(${col.name})`, _stringsToAligned(aligned));
+  const alignedCol = DG.Column.fromStrings(`msa(${col.name})`, aligned);
   alignedCol.setTag(DG.TAGS.UNITS, ''); 
   alignedCol.semType = C.SEM_TYPES.Macro_Molecule;
   return alignedCol;

package/src/utils/sequence-space.ts

@@ -0,0 +1,43 @@
+import * as DG from 'datagrok-api/dg';
+import { AvailableMetrics } from '@datagrok-libraries/ml/src/typed-metrics';
+import {reduceDimensinalityWithNormalization} from '@datagrok-libraries/ml/src/sequence-space';
+import {BitArrayMetrics, StringMetrics} from '@datagrok-libraries/ml/src/typed-metrics';
+import { Matrix } from '@datagrok-libraries/utils/src/type-declarations';
+import BitArray from '@datagrok-libraries/utils/src/bit-array';
+
+export interface ISequenceSpaceResult {
+  distance: Matrix;
+  coordinates: DG.ColumnList;
+}
+
+export async function sequenceSpace(molColumn: DG.Column, methodName: string, similarityMetric: string,
+    axes: string[], options?: any): Promise<ISequenceSpaceResult> {
+    let preparedData: any;
+    if (!(molColumn!.tags[DG.TAGS.UNITS] === 'HELM')) {
+      const sep = molColumn.getTag('separator');
+      const sepFinal = sep ? sep === '.' ? '\\\.' : sep: '-';  
+      var regex = new RegExp(sepFinal, "g");
+      if (Object.keys(AvailableMetrics['String']).includes(similarityMetric)) {
+          preparedData = molColumn.toList().map((v) => v.replace(regex, '')) as string[];
+      } else {
+          preparedData = molColumn.toList().map((v) => v.replace(regex, '')) as string[];
+      }
+    } else {
+      preparedData = molColumn.toList();
+    }
+    
+    const sequenceSpaceResult = await reduceDimensinalityWithNormalization(
+      preparedData,
+      methodName,
+      similarityMetric as StringMetrics|BitArrayMetrics,
+      options);
+    const cols: DG.Column[] = axes.map((name, index) => DG.Column.fromFloat32Array(name, sequenceSpaceResult.embedding[index]))
+    return {distance: sequenceSpaceResult.distance, coordinates: new DG.ColumnList(cols)};
+  }
+
+
+export function getEmbeddingColsNames(df: DG.DataFrame){
+    const axes = ['Embed_X', 'Embed_Y'];
+    const colNameInd = df.columns.names().filter((it) => it.includes(axes[0])).length + 1;
+    return axes.map((it) => `${it}_${colNameInd}`);
+  }