@platforma-open/milaboratories.immune-assay-data.workflow 1.11.0 → 1.12.1

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/immune-assay-data/immune-assay-data/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.immune-assay-data.workflow@1.11.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
+> @platforma-open/milaboratories.immune-assay-data.workflow@1.12.1 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/analysis.tpl.tengo"...

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @platforma-open/milaboratories.immune-assay-data.workflow
 
+## 1.12.1
+
+### Patch Changes
+
+- Updated dependencies [2eff9dc]
+  - @platforma-open/milaboratories.immune-assay-data.split-fasta@1.2.0
+
+## 1.12.0
+
+### Minor Changes
+
+- 5f43c2b: Improved scalability for large datasets
+
+### Patch Changes
+
+- Updated dependencies [5f43c2b]
+  - @platforma-open/milaboratories.immune-assay-data.coverage-mode-calc@1.3.0
+  - @platforma-open/milaboratories.immune-assay-data.merge-results@1.1.0
+  - @platforma-open/milaboratories.immune-assay-data.split-fasta@1.1.0
+
 ## 1.11.0
 
 ### Minor Changes

package/package.json

@@ -1,17 +1,19 @@
 {
   "name": "@platforma-open/milaboratories.immune-assay-data.workflow",
-  "version": "1.11.0",
+  "version": "1.12.1",
   "type": "module",
   "description": "Tengo-based template",
   "dependencies": {
     "@platforma-sdk/workflow-tengo": "5.11.0",
     "@platforma-open/soedinglab.software-mmseqs2": "1.18.3",
+    "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.3",
     "@platforma-open/milaboratories.immune-assay-data.add-header": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.2.0",
-    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0",
     "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.check-content-empty": "1.0.1"
+    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0",
+    "@platforma-open/milaboratories.immune-assay-data.check-content-empty": "1.0.1",
+    "@platforma-open/milaboratories.immune-assay-data.split-fasta": "1.2.0",
+    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.3.0",
+    "@platforma-open/milaboratories.immune-assay-data.merge-results": "1.1.0"
   },
   "devDependencies": {
     "@platforma-sdk/tengo-builder": "2.5.5"

package/src/analysis.tpl.tengo

@@ -9,10 +9,11 @@ render := import("@platforma-sdk/workflow-tengo:render")
 
 prepareFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.prepare-fasta:main")
 fastaToTsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv:main")
-addHeaderSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.add-header:main")
 covModeCalcSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc:main")
 xlsxToCsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv:main")
 checkContentEmptySw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.check-content-empty:main")
+splitFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.split-fasta:main")
+mergeResultsSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.merge-results:main")
 
 runAlignmentTpl := assets.importTemplate(":run-alignment")
 checkContentEmptyTpl := assets.importTemplate(":check-content-empty")
@@ -232,6 +233,27 @@ self.body(func(args) {
 
 	covMode := coverageModeRun.getFileContent("coverage_mode.txt")
 
+	// Split clone FASTA into 2 equal chunks to limit mmseqs2 index disk usage.
+	// Running two searches against 25M sequences each uses half the peak disk
+	// of a single 50M search. E-values are normalized to the full database size.
+	splitRun := exec.builder().
+		software(splitFastaSw).
+		mem("8GiB").
+		cpu(1).
+		addFile("clones.fasta", clonesFasta).
+		arg("-i").arg("clones.fasta").
+		arg("--chunk1").arg("chunk_1.fasta").
+		arg("--chunk2").arg("chunk_2.fasta").
+		arg("--counts").arg("counts.json").
+		saveFile("chunk_1.fasta").
+		saveFile("chunk_2.fasta").
+		saveFile("counts.json").
+		run()
+
+	chunk1Fasta := splitRun.getFile("chunk_1.fasta")
+	chunk2Fasta := splitRun.getFile("chunk_2.fasta")
+	splitCounts := splitRun.getFile("counts.json")
+
 	// MMseqs2 Alignment
 	mmseqsSearchType := "0"
     if targetSequenceType == "aminoacid" && assaySequenceType == "aminoacid" {
@@ -248,13 +270,28 @@ self.body(func(args) {
         mmseqsSearchType = "2"
     }
 
-	runMmseqs := render.create(runAlignmentTpl, {
+	runMmseqs1 := render.create(runAlignmentTpl, {
+		covMode: covMode,
+		mmseqsSearchType: mmseqsSearchType,
+		coverageThreshold: coverageThreshold,
+		identityThreshold: identityThreshold,
+		similarityType: similarityType,
+		clonesFasta: chunk1Fasta,
+		assayFasta: assayFasta,
+		lessSensitive: lessSensitive
+	}, {
+		metaInputs: {
+			mem: mem,
+			cpu: cpu
+		}
+	})
+	runMmseqs2 := render.create(runAlignmentTpl, {
 		covMode: covMode,
 		mmseqsSearchType: mmseqsSearchType,
 		coverageThreshold: coverageThreshold,
 		identityThreshold: identityThreshold,
 		similarityType: similarityType,
-		clonesFasta: clonesFasta,
+		clonesFasta: chunk2Fasta,
 		assayFasta: assayFasta,
 		lessSensitive: lessSensitive
 	}, {
@@ -264,20 +301,25 @@ self.body(func(args) {
 		}
 	})
 
-	mmseqsOutput := runMmseqs.output("mmseqsOutput")
+	mmseqsOutput1 := runMmseqs1.output("mmseqsOutput")
+	mmseqsOutput2 := runMmseqs2.output("mmseqsOutput")
 
-	// @TODO remove header stuff and replace with pt when available (!)
-	addHeaderRun := exec.builder().
-		software(addHeaderSw).
+	// Merge both raw results, add header, and normalize e-values to full database size
+	mergeRun := exec.builder().
+		software(mergeResultsSw).
 		mem("16GiB").
 		cpu(1).
-		arg("-i").arg("results.tsv").
+		addFile("results_1.tsv", mmseqsOutput1).
+		addFile("results_2.tsv", mmseqsOutput2).
+		addFile("counts.json", splitCounts).
+		arg("-i1").arg("results_1.tsv").
+		arg("-i2").arg("results_2.tsv").
+		arg("--counts").arg("counts.json").
 		arg("-o").arg("results_with_header.tsv").
-		addFile("results.tsv", mmseqsOutput).
 		saveFile("results_with_header.tsv").
 		run()
 
-	mmseqsResultTsv := addHeaderRun.getFile("results_with_header.tsv")
+	mmseqsResultTsv := mergeRun.getFile("results_with_header.tsv")
 
 	// Check if results are empty (only header line or nothing)
 	checkResultsRun := exec.builder().
@@ -297,7 +339,7 @@ self.body(func(args) {
 	emptyResults := checkResult.output("result")
 
 	result := {
-		mmseqsOutput: mmseqsOutput,
+		mmseqsOutput: mmseqsResultTsv,
 		emptyResults: emptyResults
 	}
 
@@ -315,7 +357,7 @@ self.body(func(args) {
 	)
 
 	cols := []
-	for _, col in ["bits", "evalue", "target", "pident", "alnlen", "mismatch",
+	for _, col in ["bits", "evalue", "query", "pident", "alnlen", "mismatch",
 	"gapopen", "qstart", "qend", "tstart", "tend"] {
 		cols = append(cols,
 			pt.col(col).maxBy(
@@ -325,7 +367,7 @@ self.body(func(args) {
 		)
 	}
 
-	dfRes = dfRes.groupBy("query").agg(cols...)
+	dfRes = dfRes.groupBy("target").agg(cols...)
 	// Add link column for linker pFrame (assayLinkerPframe)
 	dfRes = dfRes.withColumns(
 		pt.lit(1).cast("Int64").alias("link")
@@ -340,13 +382,13 @@ self.body(func(args) {
 
 	// import how many matches per assay sequence found
 	assayDf = assayDf.join(
-		dfRes.groupBy("target").agg(
-			pt.col("query").count().alias("queryCount")
+		dfRes.groupBy("query").agg(
+			pt.col("target").count().alias("queryCount")
 		),
 		{
 			how: "left",
 			leftOn: "seqId",
-			rightOn: "target"
+			rightOn: "query"
 		}
 	)
 	assayDf.save("assay_data.tsv")
@@ -355,7 +397,7 @@ self.body(func(args) {
 	clonesDf := dfRes.join(assayDf,
 		{
 			how: "left",
-			leftOn: "target",
+			leftOn: "query",
 			rightOn: "seqId"
 		}
 	)

package/src/build-outputs.tpl.tengo

@@ -174,7 +174,7 @@ self.body(func(inputs) {
 	cloneImportResults := xsv.importFile(
 		inputs.clonesDataTsv, "tsv", {
 			axes: [{
-				column: "query",
+				column: "target",
 				spec: inputs.datasetSpec.axesSpec[1]
 			}],
 			columns: cloneColumns,
@@ -189,11 +189,11 @@ self.body(func(inputs) {
 		inputs.bestAlignmentTsv, "tsv", {
 			axes: [
 				{
-					column: "query",
+					column: "target",
 					spec: inputs.datasetSpec.axesSpec[1]
 				},
 				{
-					column: "target",
+					column: "query",
 					spec: {
 						name: "pl7.app/vdj/assay/sequenceId",
 						type: "String",

package/src/run-alignment.tpl.tengo

@@ -32,8 +32,8 @@ self.body(func(args) {
 		mem(mem).
 		cpu(cpu).
 		arg("easy-search").
-		arg("clones.fasta").
 		arg("assay.fasta").
+		arg("clones.fasta").
 		arg("results.tsv").
 		arg("tmp").
 		arg("--threads").arg(string(cpu)).