@datagrok/bio 1.3.1 → 1.4.2

package/src/utils/multiple-sequence-alignment.ts

@@ -0,0 +1,76 @@
+import * as DG from 'datagrok-api/dg';
+
+//@ts-ignore
+import Aioli from '@biowasm/aioli';
+
+import {AlignedSequenceEncoder} from '@datagrok-libraries/bio/src/sequence-encoder';
+import * as C from './constants';
+
+/**
+ * Converts array of sequences into simple fasta string.
+ *
+ * @param {string[]} sequences Input list of sequences.
+ * @return {string} Fasta-formatted string.
+ */
+function _stringsToFasta(sequences: string[]): string {
+  return sequences.reduce((a, v, i) => a + `>sample${i + 1}\n${v}\n`, '');
+}
+
+/**
+ * Extracts array of sequences from simple fasta string.
+ *
+ * @param {string} fasta Fasta-formatted string.
+ * @return {string[]} Output list of sequences.
+ */
+function _fastaToStrings(fasta: string): string[] {
+  return fasta.replace(/>sample\d+(\r\n|\r|\n)/g, '').split('\n');
+}
+
+/**
+ * Runs Aioli environment with kalign tool.
+ *
+ * @param {DG.Column} col Column with sequences.
+ * @param {boolean} isAligned Whether the column is aligned.
+ * @return {Promise<DG.Column>} Aligned sequences.
+ */
+export async function runKalign(col: DG.Column, isAligned = false) : Promise<DG.Column> {
+  let sequences = col.toList();
+
+  if (isAligned)
+    sequences = sequences.map((v: string, _) => AlignedSequenceEncoder.clean(v).replace(/\-/g, ''));
+
+  const fasta = _stringsToFasta(sequences);
+  const CLI = await new Aioli({
+    tool: 'kalign',
+    version: '3.3.1',
+    reinit: true,
+  });
+
+  console.log(['fasta.length =', fasta.length]);
+
+  await CLI.fs.writeFile('input.fa', fasta);
+  const output = await CLI.exec('kalign input.fa -f fasta -o result.fasta');
+  const buf = await CLI.cat('result.fasta');
+
+  console.warn(output);
+
+  const aligned = _fastaToStrings(buf).slice(0, sequences.length);
+  const alignedCol = DG.Column.fromStrings(`msa(${col.name})`, aligned);
+  alignedCol.setTag(DG.TAGS.UNITS, ''); 
+  alignedCol.semType = C.SEM_TYPES.Macro_Molecule;
+  return alignedCol;
+}
+
+export async function testMSAEnoughMemory(col: DG.Column): Promise<void> {
+  const sequencesCount = col.length;
+  const delta = sequencesCount/100;
+
+  for (let i = delta; i < sequencesCount; i += delta) {
+    try {
+      await runKalign(DG.Column.fromStrings(col.name, col.toList().slice(0, Math.round(i))));
+      console.log(`runKalign succeeded on ${i}`);
+    } catch (error) {
+      console.log(`runKalign failed on ${i} with '${error}'`);
+    }
+  }
+}

package/src/utils/sequence-space.ts

@@ -0,0 +1,43 @@
+import * as DG from 'datagrok-api/dg';
+import { AvailableMetrics } from '@datagrok-libraries/ml/src/typed-metrics';
+import {reduceDimensinalityWithNormalization} from '@datagrok-libraries/ml/src/sequence-space';
+import {BitArrayMetrics, StringMetrics} from '@datagrok-libraries/ml/src/typed-metrics';
+import { Matrix } from '@datagrok-libraries/utils/src/type-declarations';
+import BitArray from '@datagrok-libraries/utils/src/bit-array';
+
+export interface ISequenceSpaceResult {
+  distance: Matrix;
+  coordinates: DG.ColumnList;
+}
+
+export async function sequenceSpace(molColumn: DG.Column, methodName: string, similarityMetric: string,
+    axes: string[], options?: any): Promise<ISequenceSpaceResult> {
+    let preparedData: any;
+    if (!(molColumn!.tags[DG.TAGS.UNITS] === 'HELM')) {
+      const sep = molColumn.getTag('separator');
+      const sepFinal = sep ? sep === '.' ? '\\\.' : sep: '-';  
+      var regex = new RegExp(sepFinal, "g");
+      if (Object.keys(AvailableMetrics['String']).includes(similarityMetric)) {
+          preparedData = molColumn.toList().map((v) => v.replace(regex, '')) as string[];
+      } else {
+          preparedData = molColumn.toList().map((v) => v.replace(regex, '')) as string[];
+      }
+    } else {
+      preparedData = molColumn.toList();
+    }
+    
+    const sequenceSpaceResult = await reduceDimensinalityWithNormalization(
+      preparedData,
+      methodName,
+      similarityMetric as StringMetrics|BitArrayMetrics,
+      options);
+    const cols: DG.Column[] = axes.map((name, index) => DG.Column.fromFloat32Array(name, sequenceSpaceResult.embedding[index]))
+    return {distance: sequenceSpaceResult.distance, coordinates: new DG.ColumnList(cols)};
+  }
+
+
+export function getEmbeddingColsNames(df: DG.DataFrame){
+    const axes = ['Embed_X', 'Embed_Y'];
+    const colNameInd = df.columns.names().filter((it) => it.includes(axes[0])).length + 1;
+    return axes.map((it) => `${it}_${colNameInd}`);
+  }

package/test-Bio-69a4761f6044-51a4ab35.html

@@ -1,245 +0,0 @@
-<html><head><meta charset="utf-8"/><title>Bio Test Report. Datagrok version datagrok/datagrok:latest SHA=69a4761f6044. Commit 51a4ab35.</title><style type="text/css">html,
-body {
-  font-family: Arial, Helvetica, sans-serif;
-  font-size: 1rem;
-  margin: 0;
-  padding: 0;
-  color: #333;
-}
-body {
-  padding: 2rem 1rem;
-  font-size: 0.85rem;
-}
-#jesthtml-content {
-  margin: 0 auto;
-  max-width: 70rem;
-}
-header {
-  display: flex;
-  align-items: center;
-}
-#title {
-  margin: 0;
-  flex-grow: 1;
-}
-#logo {
-  height: 4rem;
-}
-#timestamp {
-  color: #777;
-  margin-top: 0.5rem;
-}
-
-/** SUMMARY */
-#summary {
-  color: #333;
-  margin: 2rem 0;
-  display: flex;
-  font-family: monospace;
-  font-size: 1rem;
-}
-#summary > div {
-  margin-right: 2rem;
-  background: #eee;
-  padding: 1rem;
-  min-width: 15rem;
-}
-#summary > div:last-child {
-  margin-right: 0;
-}
-@media only screen and (max-width: 720px) {
-  #summary {
-    flex-direction: column;
-  }
-  #summary > div {
-    margin-right: 0;
-    margin-top: 2rem;
-  }
-  #summary > div:first-child {
-    margin-top: 0;
-  }
-}
-
-.summary-total {
-  font-weight: bold;
-  margin-bottom: 0.5rem;
-}
-.summary-passed {
-  color: #4f8a10;
-  border-left: 0.4rem solid #4f8a10;
-  padding-left: 0.5rem;
-}
-.summary-failed,
-.summary-obsolete-snapshots {
-  color: #d8000c;
-  border-left: 0.4rem solid #d8000c;
-  padding-left: 0.5rem;
-}
-.summary-pending {
-  color: #9f6000;
-  border-left: 0.4rem solid #9f6000;
-  padding-left: 0.5rem;
-}
-.summary-empty {
-  color: #999;
-  border-left: 0.4rem solid #999;
-}
-
-.test-result {
-  padding: 1rem;
-  margin-bottom: 0.25rem;
-}
-.test-result:last-child {
-  border: 0;
-}
-.test-result.passed {
-  background-color: #dff2bf;
-  color: #4f8a10;
-}
-.test-result.failed {
-  background-color: #ffbaba;
-  color: #d8000c;
-}
-.test-result.pending {
-  background-color: #ffdf61;
-  color: #9f6000;
-}
-
-.test-info {
-  display: flex;
-  justify-content: space-between;
-}
-.test-suitename {
-  width: 20%;
-  text-align: left;
-  font-weight: bold;
-  word-break: break-word;
-}
-.test-title {
-  width: 40%;
-  text-align: left;
-  font-style: italic;
-}
-.test-status {
-  width: 20%;
-  text-align: right;
-}
-.test-duration {
-  width: 10%;
-  text-align: right;
-  font-size: 0.75rem;
-}
-
-.failureMessages {
-  padding: 0 1rem;
-  margin-top: 1rem;
-  border-top: 1px dashed #d8000c;
-}
-.failureMessages.suiteFailure {
-  border-top: none;
-}
-.failureMsg {
-  white-space: pre-wrap;
-  white-space: -moz-pre-wrap;
-  white-space: -pre-wrap;
-  white-space: -o-pre-wrap;
-  word-wrap: break-word;
-}
-
-.suite-container {
-  margin-bottom: 2rem;
-}
-.suite-info {
-  padding: 1rem;
-  background-color: #eee;
-  color: #777;
-  display: flex;
-  align-items: center;
-  margin-bottom: 0.25rem;
-}
-.suite-info .suite-path {
-  word-break: break-all;
-  flex-grow: 1;
-  font-family: monospace;
-  font-size: 1rem;
-}
-.suite-info .suite-time {
-  margin-left: 0.5rem;
-  padding: 0.2rem 0.3rem;
-  font-size: 0.75rem;
-}
-.suite-info .suite-time.warn {
-  background-color: #d8000c;
-  color: #fff;
-}
-
-/* CONSOLE LOGS */
-.suite-consolelog {
-  margin-bottom: 0.25rem;
-  padding: 1rem;
-  background-color: #efefef;
-}
-.suite-consolelog-header {
-  font-weight: bold;
-}
-.suite-consolelog-item {
-  padding: 0.5rem;
-}
-.suite-consolelog-item pre {
-  margin: 0.5rem 0;
-  white-space: pre-wrap;
-  white-space: -moz-pre-wrap;
-  white-space: -pre-wrap;
-  white-space: -o-pre-wrap;
-  word-wrap: break-word;
-}
-.suite-consolelog-item-origin {
-  color: #777;
-  font-weight: bold;
-}
-.suite-consolelog-item-message {
-  color: #000;
-  font-size: 1rem;
-  padding: 0 0.5rem;
-}
-
-/* OBSOLETE SNAPSHOTS */
-.suite-obsolete-snapshots {
-  margin-bottom: 0.25rem;
-  padding: 1rem;
-  background-color: #ffbaba;
-  color: #d8000c;
-}
-.suite-obsolete-snapshots-header {
-  font-weight: bold;
-}
-.suite-obsolete-snapshots-item {
-  padding: 0.5rem;
-}
-.suite-obsolete-snapshots-item pre {
-  margin: 0.5rem 0;
-  white-space: pre-wrap;
-  white-space: -moz-pre-wrap;
-  white-space: -pre-wrap;
-  white-space: -o-pre-wrap;
-  word-wrap: break-word;
-}
-.suite-obsolete-snapshots-item-message {
-  color: #000;
-  font-size: 1rem;
-  padding: 0 0.5rem;
-}
-</style></head><body><div id="jesthtml-content"><header><h1 id="title">Bio Test Report. Datagrok version datagrok/datagrok:latest SHA=69a4761f6044. Commit 51a4ab35.</h1></header><div id="metadata-container"><div id="timestamp">Started: 2022-07-01 14:55:40</div><div id="summary"><div id="suite-summary"><div class="summary-total">Suites (1)</div><div class="summary-passed">1 passed</div><div class="summary-failed summary-empty">0 failed</div><div class="summary-pending summary-empty">0 pending</div></div><div id="test-summary"><div class="summary-total">Tests (1)</div><div class="summary-passed">1 passed</div><div class="summary-failed summary-empty">0 failed</div><div class="summary-pending summary-empty">0 pending</div></div></div></div><div id="suite-1" class="suite-container"><div class="suite-info"><div class="suite-path">/home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts</div><div class="suite-time warn">39.007s</div></div><div class="suite-tests"><div class="test-result passed"><div class="test-info"><div class="test-suitename"> </div><div class="test-title">TEST</div><div class="test-status">passed</div><div class="test-duration">28.74s</div></div></div></div><div class="suite-consolelog"><div class="suite-consolelog-header">Console Log</div><div class="suite-consolelog-item"><pre class="suite-consolelog-item-origin">    at Object.&lt;anonymous&gt; (/home/runner/work/public/public/packages/Bio/src/__jest__/test-node.ts:63:11)
-    at Generator.next (&lt;anonymous&gt;)
-    at fulfilled (/home/runner/work/public/public/packages/Bio/src/__jest__/test-node.ts:28:58)
-    at processTicksAndRejections (internal/process/task_queues.js:97:5)</pre><pre class="suite-consolelog-item-message">Using web root: http://localhost:8080</pre></div><div class="suite-consolelog-item"><pre class="suite-consolelog-item-origin">    at /home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts:24:11
-    at Generator.next (&lt;anonymous&gt;)
-    at /home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts:34:71
-    at new Promise (&lt;anonymous&gt;)
-    at Object.&lt;anonymous&gt;.__awaiter (/home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts:30:12)
-    at Object.&lt;anonymous&gt; (/home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts:22:23)
-    at Promise.then.completed (/home/runner/work/public/public/packages/Bio/node_modules/jest-circus/build/utils.js:391:28)
-    at new Promise (&lt;anonymous&gt;)</pre><pre class="suite-consolelog-item-message">Testing Bio package</pre></div><div class="suite-consolelog-item"><pre class="suite-consolelog-item-origin">    at /home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts:47:11
-    at Generator.next (&lt;anonymous&gt;)
-    at fulfilled (/home/runner/work/public/public/packages/Bio/src/__jest__/remote.test.ts:31:58)
-    at processTicksAndRejections (internal/process/task_queues.js:97:5)</pre><pre class="suite-consolelog-item-message"/></div></div></div></div></body></html>