@platforma-open/milaboratories.immune-assay-data.workflow 1.9.0 → 1.11.0

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/immune-assay-data/immune-assay-data/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.immune-assay-data.workflow@1.9.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
+> @platforma-open/milaboratories.immune-assay-data.workflow@1.11.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/analysis.tpl.tengo"...

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @platforma-open/milaboratories.immune-assay-data.workflow
 
+## 1.11.0
+
+### Minor Changes
+
+- ac74170: Improved performance on large datasets, eliminating disk and memory pressure
+
+## 1.10.0
+
+### Minor Changes
+
+- 29a44a2: Improved performance on large datasets
+
 ## 1.9.0
 
 ### Minor Changes

package/package.json

@@ -1,20 +1,20 @@
 {
   "name": "@platforma-open/milaboratories.immune-assay-data.workflow",
-  "version": "1.9.0",
+  "version": "1.11.0",
   "type": "module",
   "description": "Tengo-based template",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "5.10.1",
+    "@platforma-sdk/workflow-tengo": "5.11.0",
     "@platforma-open/soedinglab.software-mmseqs2": "1.18.3",
-    "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.2.0",
     "@platforma-open/milaboratories.immune-assay-data.add-header": "1.1.3",
+    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.2.0",
+    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0",
+    "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.3",
     "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.check-content-empty": "1.0.1",
-    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0"
+    "@platforma-open/milaboratories.immune-assay-data.check-content-empty": "1.0.1"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "2.4.30"
+    "@platforma-sdk/tengo-builder": "2.5.5"
   },
   "scripts": {
     "build": "rm -rf dist && pl-tengo check && pl-tengo build",

package/src/analysis.tpl.tengo

@@ -315,7 +315,7 @@ self.body(func(args) {
 	)
 
 	cols := []
-	for _, col in ["bits", "evalue", "query", "pident", "alnlen", "mismatch",
+	for _, col in ["bits", "evalue", "target", "pident", "alnlen", "mismatch",
 	"gapopen", "qstart", "qend", "tstart", "tend"] {
 		cols = append(cols,
 			pt.col(col).maxBy(
@@ -325,7 +325,7 @@ self.body(func(args) {
 		)
 	}
 
-	dfRes = dfRes.groupBy("target").agg(cols...)
+	dfRes = dfRes.groupBy("query").agg(cols...)
 	// Add link column for linker pFrame (assayLinkerPframe)
 	dfRes = dfRes.withColumns(
 		pt.lit(1).cast("Int64").alias("link")
@@ -340,13 +340,13 @@ self.body(func(args) {
 
 	// import how many matches per assay sequence found
 	assayDf = assayDf.join(
-		dfRes.groupBy("query").agg(
-			pt.col("target").count().alias("queryCount")
+		dfRes.groupBy("target").agg(
+			pt.col("query").count().alias("queryCount")
 		),
 		{
 			how: "left",
 			leftOn: "seqId",
-			rightOn: "query"
+			rightOn: "target"
 		}
 	)
 	assayDf.save("assay_data.tsv")
@@ -355,7 +355,7 @@ self.body(func(args) {
 	clonesDf := dfRes.join(assayDf,
 		{
 			how: "left",
-			leftOn: "query",
+			leftOn: "target",
 			rightOn: "seqId"
 		}
 	)

package/src/build-outputs.tpl.tengo

@@ -174,7 +174,7 @@ self.body(func(inputs) {
 	cloneImportResults := xsv.importFile(
 		inputs.clonesDataTsv, "tsv", {
 			axes: [{
-				column: "target",
+				column: "query",
 				spec: inputs.datasetSpec.axesSpec[1]
 			}],
 			columns: cloneColumns,
@@ -189,11 +189,11 @@ self.body(func(inputs) {
 		inputs.bestAlignmentTsv, "tsv", {
 			axes: [
 				{
-					column: "target",
+					column: "query",
 					spec: inputs.datasetSpec.axesSpec[1]
 				},
 				{
-					column: "query",
+					column: "target",
 					spec: {
 						name: "pl7.app/vdj/assay/sequenceId",
 						type: "String",

package/src/run-alignment.tpl.tengo

@@ -2,6 +2,7 @@ self := import("@platforma-sdk/workflow-tengo:tpl")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 exec := import("@platforma-sdk/workflow-tengo:exec")
 assets:= import("@platforma-sdk/workflow-tengo:assets")
+math := import("math")
 mmseqsSw := assets.importSoftware("@platforma-open/soedinglab.software-mmseqs2:main")
 
 self.defineOutputs("mmseqsOutput")
@@ -16,11 +17,12 @@ self.body(func(args) {
     clonesFasta := args.clonesFasta
     assayFasta := args.assayFasta
 
-	mem := "8GiB"
-	cpu := 1
+	baseMemGiB := 64
 	if !is_undefined(args.metaInputs.mem) {
-		mem = string(args.metaInputs.mem) + "GiB"
+		baseMemGiB = args.metaInputs.mem
 	}
+	mem := string(int(math.max(64, baseMemGiB))) + "GiB"
+	cpu := 1
 	if !is_undefined(args.metaInputs.cpu) {
 		cpu = args.metaInputs.cpu
 	}
@@ -30,8 +32,8 @@ self.body(func(args) {
 		mem(mem).
 		cpu(cpu).
 		arg("easy-search").
-		arg("assay.fasta").
 		arg("clones.fasta").
+		arg("assay.fasta").
 		arg("results.tsv").
 		arg("tmp").
 		arg("--threads").arg(string(cpu)).