@datagrok/bio 2.22.0 → 2.22.3

package/package.json

@@ -5,7 +5,7 @@
     "name": "Leonid Stolbov",
     "email": "lstolbov@datagrok.ai"
   },
-  "version": "2.22.0",
+  "version": "2.22.3",
   "description": "Bioinformatics support (import/export of sequences, conversion, visualization, analysis). [See more](https://github.com/datagrok-ai/public/blob/master/packages/Bio/README.md) for details.",
   "repository": {
     "type": "git",
@@ -44,18 +44,18 @@
   ],
   "dependencies": {
     "@biowasm/aioli": "^3.1.0",
-    "@datagrok-libraries/bio": "^5.53.4",
+    "@datagrok-libraries/bio": "^5.54.1",
     "@datagrok-libraries/chem-meta": "^1.2.7",
     "@datagrok-libraries/math": "^1.2.4",
     "@datagrok-libraries/ml": "^6.10.2",
     "@datagrok-libraries/tutorials": "^1.6.1",
     "@datagrok-libraries/utils": "^4.5.7",
-    "datagrok-api": "^1.25.0",
     "@webgpu/types": "^0.1.40",
     "ajv": "^8.12.0",
     "ajv-errors": "^3.0.0",
     "cash-dom": "^8.0.0",
     "css-loader": "^6.7.3",
+    "datagrok-api": "^1.25.0",
     "dayjs": "^1.11.4",
     "fastest-levenshtein": "^1.0.16",
     "openchemlib": "^7.2.3",
@@ -67,13 +67,13 @@
   "devDependencies": {
     "@datagrok-libraries/helm-web-editor": "^1.1.14",
     "@datagrok-libraries/js-draw-lite": "^0.0.10",
-    "@datagrok/chem": "^1.13.0",
+    "@datagrok/chem": "^1.15.0",
     "@datagrok/dendrogram": "^1.2.33",
     "@types/node": "^17.0.24",
     "@types/wu": "^2.1.44",
     "@typescript-eslint/eslint-plugin": "^8.8.1",
     "@typescript-eslint/parser": "^8.8.1",
-    "datagrok-tools": "latest",
+    "datagrok-tools": "^4.14.29",
     "eslint": "^8.57.1",
     "eslint-config-google": "^0.14.0",
     "eslint-plugin-rxjs": "^5.0.3",
@@ -93,7 +93,7 @@
     "debug-sequences1": "webpack && grok publish",
     "release-sequences1": "webpack && grok publish --release",
     "build-sequences1": "webpack",
-    "build": "webpack",
+    "build": " grok api && grok check --soft && webpack",
     "build-all": "npm --prefix ./../../libraries/chem-meta run build && npm --prefix ./../../js-api run build && npm --prefix ./../../libraries/utils run build && npm --prefix ./../../libraries/math run build && npm --prefix ./../../libraries/ml run build && npm --prefix ./../../libraries/bio run build && npm --prefix ./../../libraries/tutorials run build && npm run build",
     "debug-sequences1-local": "webpack && grok publish local",
     "release-sequences1-local": "webpack && grok publish local --release",

package/scripts/embed.py

@@ -7,7 +7,13 @@ from rdkit.Chem import AllChem
 from rdkit import Chem
 mol = AllChem.MolFromMolBlock(molecule) if ("M  END" in molecule) else AllChem.MolFromSmiles(molecule)
 
-AllChem.EmbedMolecule(mol, AllChem.ETKDG())
-#AllChem.UFFOptimizeMolecule(mol)
-#mol = Chem.RemoveHs(mol)
 sdf = Chem.MolToMolBlock(mol)
+try:
+    AllChem.EmbedMolecule(mol, AllChem.ETKDG())
+    AllChem.UFFOptimizeMolecule(mol)
+    mol = Chem.RemoveHs(mol)
+    sdf = Chem.MolToMolBlock(mol)
+except Exception as e:
+    pass
+# mol = Chem.RemoveHs(mol)
+

package/src/analysis/sequence-diversity-viewer.ts

@@ -13,13 +13,17 @@ import {MmDistanceFunctionsNames} from '@datagrok-libraries/ml/src/macromolecule
 import {DistanceMatrixService, dmLinearIndex} from '@datagrok-libraries/ml/src/distance-matrix';
 import {MmcrTemps} from '@datagrok-libraries/bio/src/utils/cell-renderer-consts';
 
+const MAX_SAMPLE_SIZE = 10000;
+
 export class SequenceDiversityViewer extends SequenceSearchBaseViewer {
-  diverseColumnLabel: string | null; // Use postfix Label to prevent activating table column selection editor
+  diverseColumnLabel: string | null;
 
   renderMolIds: number[] | null = null;
   columnNames = [];
   computeCompleted = new Subject<boolean>();
 
+  private sampledIndices: number[] | null = null;
+
   constructor(
     private readonly seqHelper: ISeqHelper,
   ) {
@@ -32,8 +36,7 @@ export class SequenceDiversityViewer extends SequenceSearchBaseViewer {
       return;
     if (this.dataFrame) {
       if (computeData && this.targetColumn) {
-        const sh = this.seqHelper.getSeqHandler(this.targetColumn);
-        await (sh.isFasta() ? this.computeByMM() : this.computeByChem());
+        await this.computeByMM();
 
         const diverseColumnName: string = this.diverseColumnLabel != null ? this.diverseColumnLabel :
           `diverse (${this.targetColumnName})`;
@@ -57,30 +60,89 @@ export class SequenceDiversityViewer extends SequenceSearchBaseViewer {
     }
   }
 
-  private async computeByChem() {
-    const monomericMols = await getMonomericMols(this.targetColumn!, this.seqHelper);
-    //need to create df to calculate fingerprints
-    const _monomericMolsDf = DG.DataFrame.fromColumns([monomericMols]);
-    this.renderMolIds = await grok.functions.call('Chem:callChemDiversitySearch', {
-      col: monomericMols,
-      metricName: this.distanceMetric,
-      limit: this.limit,
-      fingerprint: this.fingerprint,
-    });
-  }
-
   private async computeByMM() {
-    const encodedSequences =
-      (await getEncodedSeqSpaceCol(this.targetColumn!, MmDistanceFunctionsNames.LEVENSHTEIN)).seqList;
+    const totalLength = this.targetColumn!.length;
+    let workingIndices: number[];
+
+    // Determine if we need to sample the data
+    if (this.requiresSampling && totalLength > MAX_SAMPLE_SIZE) {
+      workingIndices = this.createRandomSample(totalLength, MAX_SAMPLE_SIZE);
+      this.sampledIndices = workingIndices;
+    } else {
+      workingIndices = Array.from({length: totalLength}, (_, i) => i);
+      this.sampledIndices = null;
+    }
+
+    const distanceFunction = this.distanceMetric as MmDistanceFunctionsNames;
+
+    // Call with individual parameters instead of params object
+    const encodedResult = await getEncodedSeqSpaceCol(
+      this.targetColumn!,
+      distanceFunction,
+      this.fingerprint,
+      this.gapOpen,
+      this.gapExtend
+    );
+    const fullEncodedSequences = encodedResult.seqList;
+    const options = encodedResult.options;
+
+    // Extract only the sequences we need for the working set
+    const workingEncodedSequences = workingIndices.map((idx) => fullEncodedSequences[idx]);
+
     const distanceMatrixService = new DistanceMatrixService(true, false);
-    const distanceMatrixData = await distanceMatrixService.calc(encodedSequences, MmDistanceFunctionsNames.LEVENSHTEIN);
+    const distanceMatrixData = await distanceMatrixService.calc(
+      workingEncodedSequences,
+      distanceFunction,
+      true, // normalize
+      options
+    );
     distanceMatrixService.terminate();
-    const len = this.targetColumn!.length;
-    const linearizeFunc = dmLinearIndex(len);
-    this.renderMolIds = getDiverseSubset(len, Math.min(len, this.limit),
+
+    const workingLength = workingIndices.length;
+    const linearizeFunc = dmLinearIndex(workingLength);
+
+    const diverseIndicesInWorkingSet = getDiverseSubset(
+      workingLength,
+      Math.min(workingLength, this.limit),
       (i1: number, i2: number) => {
-        return this.targetColumn!.isNone(i1) || this.targetColumn!.isNone(i2) ? 0 :
-          distanceMatrixData[linearizeFunc(i1, i2)];
-      });
+        return distanceMatrixData[linearizeFunc(i1, i2)];
+      }
+    );
+
+    // Map back to original indices
+    this.renderMolIds = diverseIndicesInWorkingSet.map((workingIndex) => workingIndices[workingIndex]);
   }
+
+  private createRandomSample(totalLength: number, sampleSize: number): number[] {
+    const validIndices: number[] = [];
+    for (let i = 0; i < totalLength; i++) {
+      if (!this.targetColumn!.isNone(i))
+        validIndices.push(i);
+    }
+
+    if (validIndices.length <= sampleSize)
+      return validIndices;
+
+    for (let i = validIndices.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      const temp = validIndices[i];
+      validIndices[i] = validIndices[j];
+      validIndices[j] = temp;
+    }
+
+    return validIndices.slice(0, sampleSize);
+  }
+
+  // // Helper method to get information about sampling (useful for debugging/info)
+  // getSamplingInfo(): {isSampled: boolean, originalSize?: number, sampleSize?: number, sampledIndices?: number[]} {
+  //   if (this.sampledIndices) {
+  //     return {
+  //       isSampled: true,
+  //       originalSize: this.targetColumn?.length,
+  //       sampleSize: this.sampledIndices.length,
+  //       sampledIndices: this.sampledIndices
+  //     };
+  //   }
+  //   return {isSampled: false};
+  // }
 }

package/src/analysis/sequence-search-base-viewer.ts

@@ -2,30 +2,62 @@ import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
 import * as grok from 'datagrok-api/grok';
 
-import {CHEM_SIMILARITY_METRICS} from '@datagrok-libraries/ml/src/distance-metrics-methods';
+import {MmDistanceFunctionsNames} from '@datagrok-libraries/ml/src/macromolecule-distance-functions';
 import {TAGS as bioTAGS} from '@datagrok-libraries/bio/src/utils/macromolecule';
 import {SearchBaseViewer} from '@datagrok-libraries/ml/src/viewers/search-base-viewer';
 
-const MAX_ROWS_FOR_DISTANCE_MATRIX = 22000;
+const MAX_ROWS_FOR_DISTANCE_MATRIX = 10000;
 
 export class SequenceSearchBaseViewer extends SearchBaseViewer {
   distanceMetric: string;
   fingerprint: string;
-  metricsProperties = ['distanceMetric', 'fingerprint'];
-  fingerprintChoices = ['Morgan', 'Pattern'];
+  gapOpen: number;
+  gapExtend: number;
+
+  metricsProperties = ['distanceMetric', 'fingerprint', 'gapOpen', 'gapExtend'];
+  fingerprintChoices = ['Morgan', 'RDKit', 'Pattern', 'AtomPair', 'MACCS', 'TopologicalTorsion'];
+  distanceFunctionChoices = [
+    MmDistanceFunctionsNames.NEEDLEMANN_WUNSCH,
+    MmDistanceFunctionsNames.HAMMING,
+    MmDistanceFunctionsNames.LEVENSHTEIN,
+    MmDistanceFunctionsNames.MONOMER_CHEMICAL_DISTANCE
+  ];
+
   tags = [DG.TAGS.UNITS, bioTAGS.aligned, bioTAGS.separator, bioTAGS.alphabet, 'cell.renderer'];
   preComputeDistanceMatrix: boolean = false;
+  requiresSampling: boolean = false;
 
   constructor(name: string, semType: string) {
     super(name, semType);
-    this.fingerprint = this.string('fingerprint', this.fingerprintChoices[0], {choices: this.fingerprintChoices});
-    this.distanceMetric = this.string('distanceMetric', CHEM_SIMILARITY_METRICS[0], {choices: CHEM_SIMILARITY_METRICS});
+
+    this.distanceMetric = this.string('distanceMetric', MmDistanceFunctionsNames.HAMMING, {
+      choices: this.distanceFunctionChoices
+    });
+
+    this.fingerprint = this.string('fingerprint', this.fingerprintChoices[0], {
+      choices: this.fingerprintChoices
+    });
+
+    this.gapOpen = this.float('gapOpen', 1);
+    this.gapExtend = this.float('gapExtend', 0.6);
   }
 
   async onTableAttached(): Promise<void> {
     super.onTableAttached();
 
-    if (this.dataFrame)
-      this.preComputeDistanceMatrix = this.dataFrame.rowCount <= MAX_ROWS_FOR_DISTANCE_MATRIX;
+    if (this.dataFrame) {
+      const rowCount = this.dataFrame.rowCount;
+      this.preComputeDistanceMatrix = rowCount <= MAX_ROWS_FOR_DISTANCE_MATRIX;
+      this.requiresSampling = rowCount > MAX_ROWS_FOR_DISTANCE_MATRIX;
+    }
+  }
+
+  needsGapPenalties(): boolean {
+    return this.distanceMetric === MmDistanceFunctionsNames.NEEDLEMANN_WUNSCH;
+  }
+
+  needsFingerprint(): boolean {
+    return this.distanceMetric === MmDistanceFunctionsNames.MONOMER_CHEMICAL_DISTANCE ||
+           this.distanceMetric === MmDistanceFunctionsNames.NEEDLEMANN_WUNSCH;
   }
 }

package/src/analysis/sequence-similarity-viewer.ts

@@ -1,3 +1,5 @@
+/* eslint-disable max-len */
+/* eslint-disable space-infix-ops */
 import * as grok from 'datagrok-api/grok';
 import * as ui from 'datagrok-api/ui';
 import * as DG from 'datagrok-api/dg';
@@ -34,6 +36,12 @@ export class SequenceSimilarityViewer extends SequenceSearchBaseViewer {
   analysisGrid?: DG.Grid;
   subInited: boolean = false;
 
+  // Track last parameters to avoid unnecessary recomputation
+  private lastDistanceMetric: string = '';
+  private lastFingerprint: string = '';
+  private lastGapOpen: number = 0;
+  private lastGapExtend: number = 0;
+
   constructor(
     private readonly seqHelper: ISeqHelper,
     demo?: boolean,
@@ -55,10 +63,16 @@ export class SequenceSimilarityViewer extends SequenceSearchBaseViewer {
       return;
     if (this.targetColumn) {
       this.curIdx = this.dataFrame!.currentRowIdx == -1 ? 0 : this.dataFrame!.currentRowIdx;
-      if (computeData && !this.gridSelect) {
-        this.targetMoleculeIdx = (this.dataFrame!.currentRowIdx ?? -1) < 0 ? 0 : this.dataFrame!.currentRowIdx;
 
-        await this.computeByMM();
+      // Force recomputation if parameters changed
+      const parametersChanged =
+        this.lastDistanceMetric !== this.distanceMetric ||
+        this.lastFingerprint !== this.fingerprint ||
+        this.lastGapOpen !== this.gapOpen ||
+        this.lastGapExtend !== this.gapExtend;
+
+      if ((computeData && !this.gridSelect) || parametersChanged) {
+        this.targetMoleculeIdx = (this.dataFrame!.currentRowIdx ?? -1) < 0 ? 0 : this.dataFrame!.currentRowIdx; await this.computeByMM();
         const similarColumnName: string = this.similarColumnLabel != null ? this.similarColumnLabel :
           `similar (${this.targetColumn})`;
         this.molCol = DG.Column.string(similarColumnName,
@@ -106,14 +120,40 @@ export class SequenceSimilarityViewer extends SequenceSearchBaseViewer {
   private async computeByMM() {
     const len = this.targetColumn!.length;
     const actualLimit = Math.min(this.limit, len - 1);
-    if (!this.knn || this.kPrevNeighbors !== actualLimit) {
-      const encodedSequences =
-        (await getEncodedSeqSpaceCol(this.targetColumn!, MmDistanceFunctionsNames.LEVENSHTEIN)).seqList;
 
+    // Check if need to recalculate knn due to parameter changes
+    const needsRecalculation = !this.knn ||
+      this.kPrevNeighbors !== actualLimit ||
+      this.lastDistanceMetric !== this.distanceMetric ||
+      this.lastFingerprint !== this.fingerprint ||
+      this.lastGapOpen !== this.gapOpen ||
+      this.lastGapExtend !== this.gapExtend;
+
+    if (needsRecalculation) {
+      const distanceFunction = this.distanceMetric as MmDistanceFunctionsNames;
+
+      // Call with individual parameters instead of params object
+      const encodedResult = await getEncodedSeqSpaceCol(
+        this.targetColumn!,
+        distanceFunction,
+        this.fingerprint,
+        this.gapOpen,
+        this.gapExtend
+      );
+      const encodedSequences = encodedResult.seqList;
+      const options = encodedResult.options;
+
+      // Store current parameters for next comparison
+      this.lastDistanceMetric = this.distanceMetric;
+      this.lastFingerprint = this.fingerprint;
+      this.lastGapOpen = this.gapOpen;
+      this.lastGapExtend = this.gapExtend;
       this.kPrevNeighbors = actualLimit;
+
       this.knn = await (new SparseMatrixService()
-        .getKNN(encodedSequences, MmDistanceFunctionsNames.LEVENSHTEIN, Math.min(this.limit, len - 1)));
+        .getKNN(encodedSequences, distanceFunction, actualLimit, options));
     }
+
     const indexWScore = new Array(actualLimit).fill(0).map((_, i) => ({
       idx: this.knn!.knnIndexes[this.targetMoleculeIdx][i],
       score: 1 - this.knn!.knnDistances[this.targetMoleculeIdx][i],

package/src/analysis/sequence-space.ts

@@ -15,9 +15,13 @@ export interface ISequenceSpaceResult {
 }
 
 export async function getEncodedSeqSpaceCol(
-  seqCol: DG.Column, similarityMetric: BitArrayMetrics | MmDistanceFunctionsNames, fingerprintType: string = 'Morgan'
+  seqCol: DG.Column,
+  similarityMetric: BitArrayMetrics | MmDistanceFunctionsNames,
+  fingerprintType: string = 'Morgan',
+  gapOpen: number = 1,
+  gapExtend: number = 0.6
 ): Promise<{ seqList: string[], options: { [_: string]: any } }> {
-// encodes sequences using utf characters to also support multichar and non fasta sequences
+  // encodes sequences using utf characters to also support multichar and non fasta sequences
   const rowCount = seqCol.length;
   const sh = _package.seqHelper.getSeqHandler(seqCol);
   const encList = Array<string>(rowCount);
@@ -25,6 +29,7 @@ export async function getEncodedSeqSpaceCol(
   const charCodeMap = new Map<string, string>();
   const seqColCats = seqCol.categories;
   const seqColRawData = seqCol.getRawData();
+
   for (let rowIdx = 0; rowIdx < rowCount; rowIdx++) {
     const catI = seqColRawData[rowIdx];
     const seq = seqColCats[catI];
@@ -44,7 +49,10 @@ export async function getEncodedSeqSpaceCol(
       encList[rowIdx] += charCodeMap.get(char)!;
     }
   }
+
   let options = {} as mmDistanceFunctionArgs;
+
+  // Handle fingerprint-based distance functions
   if (
     similarityMetric === MmDistanceFunctionsNames.MONOMER_CHEMICAL_DISTANCE ||
     similarityMetric === MmDistanceFunctionsNames.NEEDLEMANN_WUNSCH
@@ -56,9 +64,21 @@ export async function getEncodedSeqSpaceCol(
     Object.entries(monomerRes.alphabetIndexes).forEach(([key, value]) => {
       monomerHashToMatrixMap[charCodeMap.get(key)!] = value;
     });
+
     // sets distance function args in place.
-    const maxLength = encList.reduce((acc, val) => Math.max(acc, val.length), 0);
-    options = {scoringMatrix: monomerRes.scoringMatrix, alphabetIndexes: monomerHashToMatrixMap, maxLength};
+    const maxLength = encList.reduce((acc, val) => Math.max(acc, val?.length || 0), 0);
+    options = {
+      scoringMatrix: monomerRes.scoringMatrix,
+      alphabetIndexes: monomerHashToMatrixMap,
+      maxLength
+    } as mmDistanceFunctionArgs;
+
+    // Add gap penalties only for Needleman-Wunsch
+    if (similarityMetric === MmDistanceFunctionsNames.NEEDLEMANN_WUNSCH) {
+      (options as any).gapOpen = gapOpen;
+      (options as any).gapExtend = gapExtend;
+    }
   }
+
   return {seqList: encList, options};
 }

package/src/demo/bio05-helm-msa-sequence-space.ts

@@ -77,7 +77,7 @@ export async function demoBio05UI(): Promise<void> {
           const method: string = pepseaMethods[0];
           const gapOpen: number = 1.53;
           const gapExtend: number = 0;
-          msaHelmCol = (await runPepsea(helmCol, msaHelmColName, method, gapOpen, gapExtend, undefined))!;
+          msaHelmCol = (await runPepsea(df, helmCol, msaHelmColName, method, gapOpen, gapExtend, undefined))!;
           if (!msaHelmCol)
             throw new Error(`Empty MSA result.`);
           df.columns.add(msaHelmCol);