@platforma-open/milaboratories.mixcr-amplicon-alignment.workflow 1.4.0 → 1.5.0

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/mixcr-amplicon-alignment/mixcr-amplicon-alignment/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.mixcr-amplicon-alignment.workflow@1.4.0 build /home/runner/work/mixcr-amplicon-alignment/mixcr-amplicon-alignment/workflow
+> @platforma-open/milaboratories.mixcr-amplicon-alignment.workflow@1.5.0 build /home/runner/work/mixcr-amplicon-alignment/mixcr-amplicon-alignment/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
   info: Skipping unknown file type: wf.test.ts

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @platforma-open/milaboratories.mixcr-amplicon-alignment.workflow
 
+## 1.5.0
+
+### Minor Changes
+
+- af0cbb1: Support batch system
+
 ## 1.4.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-open/milaboratories.mixcr-amplicon-alignment.workflow",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "MiXCR Amplicon Alignment Workflow",
   "type": "module",
   "dependencies": {

package/src/aggregate-by-clonotype-key.tpl.tengo

@@ -3,9 +3,11 @@ self := import("@platforma-sdk/workflow-tengo:tpl")
 pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 slices := import("@platforma-sdk/workflow-tengo:slices")
 maps := import("@platforma-sdk/workflow-tengo:maps")
+units := import("@platforma-sdk/workflow-tengo:units")
 pt := import("@platforma-sdk/workflow-tengo:pt")
 clonotypeLabel := import(":clonotype-label")
 
+math := import("math")
 json := import("json")
 
 self.defineOutputs("tsv")
@@ -24,10 +26,15 @@ self.body(func(inputs) {
 	schemaPerClonotypeNoAggregates := params.schemaPerClonotypeNoAggregates
 	schemaPerSample := params.schemaPerSample
 
-	wf := pt.workflow()
-	dataFrames := []
-
 	inputMap := inputData.inputs()
+	numberOfSamples := len(inputMap)
+
+	wf := pt.workflow().
+		inMediumQueue().
+		mem(int(math.max(numberOfSamples, 64)) * units.GB).
+		cpu(int(math.max(numberOfSamples, 32)))
+
+	dataFrames := []
 
 	baseSchemaForRead := schemaPerSample + [ { column: "clonotypeKey", type: "String" } ]
 

package/src/main.tpl.tengo

@@ -30,6 +30,8 @@ wf.body(func(args) {
 	fivePrimePrimer := args.fivePrimePrimer
 	chains := args.chains
 	limitInput := args.limitInput
+	perProcessMemGB := args.perProcessMemGB
+	perProcessCPUs := args.perProcessCPUs
 
 	// Create V and J gene FASTA files
 	
@@ -81,6 +83,8 @@ wf.body(func(args) {
 
 		params: smart.createJsonResource({
 			blockId: blockId,
+			perProcessMemGB: perProcessMemGB,
+			perProcessCPUs: perProcessCPUs,
 			threePrimePrimer: threePrimePrimer,
 			fivePrimePrimer: fivePrimePrimer,
 			chains: chains,

package/src/mixcr-analyze.tpl.tengo

@@ -27,6 +27,8 @@ self.body(func(inputs) {
 	params := inputs.params
 	fileExtension := params.fileExtension
 	limitInput := inputs.limitInput
+	perProcessMemGB := inputs.perProcessMemGB
+	perProcessCPUs := inputs.perProcessCPUs
 	ll.print("__THE_LOG__" + string(limitInput))
 
 	threePrimePrimer := inputs.threePrimePrimer
@@ -78,6 +80,20 @@ self.body(func(inputs) {
 		// 	mixcrCmdBuilder.arg("--rigid-right-alignment-boundary").arg("J")
 		// }
 
+	// CPUs number per process
+	if !is_undefined(perProcessCPUs) {
+	    mixcrCmdBuilder.cpu(perProcessCPUs)
+	} else {
+	    mixcrCmdBuilder.cpu(16)     // default CPUs number per sample
+	}
+	// Memory limit per process
+    if !is_undefined(perProcessMemGB) {
+        // memory per process was set in "Advanced Settings" block directly
+		mixcrCmdBuilder.mem(string(perProcessMemGB) + "GB")
+	} else {
+		mixcrCmdBuilder.mem("64GB")
+	}
+
 	if !is_undefined(limitInput) {
 		mixcrCmdBuilder.arg("--limit-input").arg(string(limitInput))
 	}

package/src/mixcr-export.tpl.tengo

@@ -32,6 +32,8 @@ self.body(func(inputs) {
 	createExport := func(additionalAction) {
 		mixcrCmdBuilder := exec.builder().
 			inMediumQueue().
+			mem("12GB").
+		    cpu(2).
 			printErrStreamToStdout().
 			dontSaveStdoutOrStderr().
 			software(mixcrSw).
@@ -70,7 +72,11 @@ self.body(func(inputs) {
 	}
 
 	// Simplified PTabler processing for main TSV output
-	wfMain := pt.workflow()
+	wfMain := pt.workflow().
+		inMediumQueue().
+		mem("8GB").
+		cpu(2)
+		
 	frameInputMap := {
 		file: unprocessedTsv,
 		xsvType: "tsv",

package/src/process.tpl.tengo

@@ -32,6 +32,8 @@ self.body(func(inputs) {
 	chains := params.chains
 	mixcrChains := params.mixcrChains
 	limitInput := inputs.limitInput
+	perProcessMemGB := params.perProcessMemGB
+	perProcessCPUs := params.perProcessCPUs
 	fileExtension := inputSpec.domain["pl7.app/fileExtension"]
 
 	// Use calculateExportSpecs for output columns
@@ -161,7 +163,14 @@ self.body(func(inputs) {
 					referenceLibrary: referenceLibrary
 				},
 				limitInput: limitInput
-			}
+			},
+
+			// by passing those parameters as meta fields we allow for recovery and deduplication mechanisms
+			// to pick up the results from executions with different values for CPU and Memory overrides
+            metaExtra: {
+                perProcessMemGB: perProcessMemGB,
+                perProcessCPUs: perProcessCPUs
+            }
 		}
 	)
 
@@ -190,6 +199,8 @@ self.body(func(inputs) {
 			storageFormat: "Binary",
 			partitionKeyLength: 0
 		},
+		mem: "16GB",
+		cpu: 2,
 		name: "byCloneKeyBySample",
 		path: ["tsv"]
 	}]
@@ -231,6 +242,8 @@ self.body(func(inputs) {
 			columns: columnsSpecPerClonotypeNoAggregates + columnsSpecPerClonotypeAggregates,
 			storageFormat: "Binary"
 		},
+		mem: "12GB",
+		cpu: 2,
 		name: "aggregates",
 		path: ["tsv"]
 	} ]

package/src/repseqio-library.tpl.tengo

@@ -27,7 +27,8 @@ self.body(func(inputs) {
 		arg("--gene-feature").arg("VRegion").
 		arg("--name-index").arg("0").
 		arg("vGene.fasta").addFile("vGene.fasta", vGeneFasta).
-		arg("vGene.json").saveFile("vGene.json")
+		arg("vGene.json").saveFile("vGene.json").
+		cpu(1).mem("4GB")
 	
 	// Run repseqio command
 	repseqioVgeneResult := repseqioVgeneCmd.run()
@@ -45,7 +46,8 @@ self.body(func(inputs) {
 		arg("--gene-feature").arg("JRegion").
 		arg("--name-index").arg("0").
 		arg("jGene.fasta").addFile("jGene.fasta", jGeneFasta).
-		arg("jGene.json").saveFile("jGene.json")
+		arg("jGene.json").saveFile("jGene.json").
+		cpu(1).mem("4GB")
 	
 	repseqioJgeneResult := repseqioJgeneCmd.run()
 	referenceLibraryJgene := repseqioJgeneResult.getFile("jGene.json")
@@ -58,7 +60,8 @@ self.body(func(inputs) {
 		arg("vGene.json").
 		arg("jGene.json").
 		arg("referenceLibrary.json").
-		saveFile("referenceLibrary.json")
+		saveFile("referenceLibrary.json").
+		cpu(1).mem("4GB")
 	
 	repseqioMergeResult := repseqioMergeCmd.run()
 	referenceLibrary := repseqioMergeResult.getFile("referenceLibrary.json")
@@ -73,7 +76,8 @@ self.body(func(inputs) {
 		arg("referenceLibrary.json").
 		arg("referenceLibrary.json").
 		saveFile("referenceLibrary.json").
-		addFile("vGene.fasta", vGeneFasta)
+		addFile("vGene.fasta", vGeneFasta).
+		cpu(1).mem("4GB")
 	
 	repseqioInferPointsVResult := repseqioInferPointsVCmd.run()
 	referenceLibraryV := repseqioInferPointsVResult.getFile("referenceLibrary.json")
@@ -88,7 +92,8 @@ self.body(func(inputs) {
 		arg("referenceLibrary.json").
 		arg("referenceLibrary.json").
 		saveFile("referenceLibrary.json").
-		addFile("jGene.fasta", jGeneFasta)
+		addFile("jGene.fasta", jGeneFasta).
+		cpu(1).mem("4GB")
 	
 	repseqioInferPointsJResult := repseqioInferPointsJCmd.run()
 	referenceLibraryJ := repseqioInferPointsJResult.getFile("referenceLibrary.json")
@@ -102,7 +107,8 @@ self.body(func(inputs) {
 		arg("referenceLibrary.json").
 		saveFile("referenceLibrary.json").
 		addFile("vGene.fasta", vGeneFasta).
-		addFile("jGene.fasta", jGeneFasta)
+		addFile("jGene.fasta", jGeneFasta).
+		cpu(1).mem("4GB")
 	
 	repseqioCompileResult := repseqioCompileCmd.run()
 	referenceLibraryCompiled := repseqioCompileResult.getFile("referenceLibrary.json")
@@ -113,7 +119,8 @@ self.body(func(inputs) {
 		addFile("referenceLibrary.json", referenceLibraryCompiled).
 		arg("referenceLibrary.json").
 		addFile("vGene.fasta", vGeneFasta).
-		addFile("jGene.fasta", jGeneFasta)
+		addFile("jGene.fasta", jGeneFasta).
+		cpu(1).mem("4GB")
 	
 	repseqioDebugResult := repseqioDebugCmd.run()
 	debugOutput := repseqioDebugResult.getStdoutStream()