@platforma-open/milaboratories.immune-assay-data.workflow 1.10.0 → 1.12.0

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/immune-assay-data/immune-assay-data/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.immune-assay-data.workflow@1.10.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
+> @platforma-open/milaboratories.immune-assay-data.workflow@1.12.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/analysis.tpl.tengo"...

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @platforma-open/milaboratories.immune-assay-data.workflow
 
+## 1.12.0
+
+### Minor Changes
+
+- 5f43c2b: Improved scalability for large datasets
+
+### Patch Changes
+
+- Updated dependencies [5f43c2b]
+  - @platforma-open/milaboratories.immune-assay-data.coverage-mode-calc@1.3.0
+  - @platforma-open/milaboratories.immune-assay-data.merge-results@1.1.0
+  - @platforma-open/milaboratories.immune-assay-data.split-fasta@1.1.0
+
+## 1.11.0
+
+### Minor Changes
+
+- ac74170: Improved performance on large datasets, eliminating disk and memory pressure
+
 ## 1.10.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-open/milaboratories.immune-assay-data.workflow",
-  "version": "1.10.0",
+  "version": "1.12.0",
   "type": "module",
   "description": "Tengo-based template",
   "dependencies": {
@@ -8,8 +8,10 @@
     "@platforma-open/soedinglab.software-mmseqs2": "1.18.3",
     "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.3",
     "@platforma-open/milaboratories.immune-assay-data.add-header": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.2.0",
+    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.3.0",
     "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.3",
+    "@platforma-open/milaboratories.immune-assay-data.merge-results": "1.1.0",
+    "@platforma-open/milaboratories.immune-assay-data.split-fasta": "1.1.0",
     "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0",
     "@platforma-open/milaboratories.immune-assay-data.check-content-empty": "1.0.1"
   },

package/src/analysis.tpl.tengo

@@ -9,10 +9,11 @@ render := import("@platforma-sdk/workflow-tengo:render")
 
 prepareFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.prepare-fasta:main")
 fastaToTsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv:main")
-addHeaderSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.add-header:main")
 covModeCalcSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc:main")
 xlsxToCsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv:main")
 checkContentEmptySw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.check-content-empty:main")
+splitFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.split-fasta:main")
+mergeResultsSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.merge-results:main")
 
 runAlignmentTpl := assets.importTemplate(":run-alignment")
 checkContentEmptyTpl := assets.importTemplate(":check-content-empty")
@@ -232,6 +233,27 @@ self.body(func(args) {
 
 	covMode := coverageModeRun.getFileContent("coverage_mode.txt")
 
+	// Split clone FASTA into 2 equal chunks to limit mmseqs2 index disk usage.
+	// Running two searches against 25M sequences each uses half the peak disk
+	// of a single 50M search. E-values are normalized to the full database size.
+	splitRun := exec.builder().
+		software(splitFastaSw).
+		mem("8GiB").
+		cpu(1).
+		addFile("clones.fasta", clonesFasta).
+		arg("-i").arg("clones.fasta").
+		arg("--chunk1").arg("chunk_1.fasta").
+		arg("--chunk2").arg("chunk_2.fasta").
+		arg("--counts").arg("counts.json").
+		saveFile("chunk_1.fasta").
+		saveFile("chunk_2.fasta").
+		saveFile("counts.json").
+		run()
+
+	chunk1Fasta := splitRun.getFile("chunk_1.fasta")
+	chunk2Fasta := splitRun.getFile("chunk_2.fasta")
+	splitCounts := splitRun.getFile("counts.json")
+
 	// MMseqs2 Alignment
 	mmseqsSearchType := "0"
     if targetSequenceType == "aminoacid" && assaySequenceType == "aminoacid" {
@@ -248,13 +270,28 @@ self.body(func(args) {
         mmseqsSearchType = "2"
     }
 
-	runMmseqs := render.create(runAlignmentTpl, {
+	runMmseqs1 := render.create(runAlignmentTpl, {
+		covMode: covMode,
+		mmseqsSearchType: mmseqsSearchType,
+		coverageThreshold: coverageThreshold,
+		identityThreshold: identityThreshold,
+		similarityType: similarityType,
+		clonesFasta: chunk1Fasta,
+		assayFasta: assayFasta,
+		lessSensitive: lessSensitive
+	}, {
+		metaInputs: {
+			mem: mem,
+			cpu: cpu
+		}
+	})
+	runMmseqs2 := render.create(runAlignmentTpl, {
 		covMode: covMode,
 		mmseqsSearchType: mmseqsSearchType,
 		coverageThreshold: coverageThreshold,
 		identityThreshold: identityThreshold,
 		similarityType: similarityType,
-		clonesFasta: clonesFasta,
+		clonesFasta: chunk2Fasta,
 		assayFasta: assayFasta,
 		lessSensitive: lessSensitive
 	}, {
@@ -264,20 +301,25 @@ self.body(func(args) {
 		}
 	})
 
-	mmseqsOutput := runMmseqs.output("mmseqsOutput")
+	mmseqsOutput1 := runMmseqs1.output("mmseqsOutput")
+	mmseqsOutput2 := runMmseqs2.output("mmseqsOutput")
 
-	// @TODO remove header stuff and replace with pt when available (!)
-	addHeaderRun := exec.builder().
-		software(addHeaderSw).
+	// Merge both raw results, add header, and normalize e-values to full database size
+	mergeRun := exec.builder().
+		software(mergeResultsSw).
 		mem("16GiB").
 		cpu(1).
-		arg("-i").arg("results.tsv").
+		addFile("results_1.tsv", mmseqsOutput1).
+		addFile("results_2.tsv", mmseqsOutput2).
+		addFile("counts.json", splitCounts).
+		arg("-i1").arg("results_1.tsv").
+		arg("-i2").arg("results_2.tsv").
+		arg("--counts").arg("counts.json").
 		arg("-o").arg("results_with_header.tsv").
-		addFile("results.tsv", mmseqsOutput).
 		saveFile("results_with_header.tsv").
 		run()
 
-	mmseqsResultTsv := addHeaderRun.getFile("results_with_header.tsv")
+	mmseqsResultTsv := mergeRun.getFile("results_with_header.tsv")
 
 	// Check if results are empty (only header line or nothing)
 	checkResultsRun := exec.builder().
@@ -297,7 +339,7 @@ self.body(func(args) {
 	emptyResults := checkResult.output("result")
 
 	result := {
-		mmseqsOutput: mmseqsOutput,
+		mmseqsOutput: mmseqsResultTsv,
 		emptyResults: emptyResults
 	}
 

package/src/run-alignment.tpl.tengo

@@ -27,10 +27,6 @@ self.body(func(args) {
 		cpu = args.metaInputs.cpu
 	}
 
-	// Cap mmseqs2 in-RAM usage to 80% of allocated memory so it splits to disk
-	// rather than getting OOM-killed by the kernel on large datasets.
-	memLimit := "{int(ceil(system.ram.gb * 0.8))}" + "G"
-
 	mmseqs := exec.builder().
 		software(mmseqsSw).
 		mem(mem).
@@ -40,7 +36,6 @@ self.body(func(args) {
 		arg("clones.fasta").
 		arg("results.tsv").
 		arg("tmp").
-		arg("--split-memory-limit").argWithVar(memLimit).
 		arg("--threads").arg(string(cpu)).
 		arg("--max-seqs").arg("10000").
 		arg("--search-type").arg(mmseqsSearchType).