@platforma-open/milaboratories.mixcr-shm-trees.workflow 3.3.0 → 3.4.0

package/src/main.tpl.tengo

@@ -45,6 +45,7 @@ wf.body(func(args) {
 			trees: results.output("trees"),
 			treeNodes: results.output("treeNodes"),
 			treeNodesWithClones: results.output("treeNodesWithClones"),
+			treeNodesUniqueIsotype: results.output("treeNodesUniqueIsotype"),
 
 			soiNodesResults: results.output("soiNodesResults"),
 			soiTreesResults: results.output("soiTreesResults"),

package/src/process.tpl.tengo

@@ -32,6 +32,7 @@ self.body(func(inputs) {
 		hasCellTags: false,
 		// will be filled
 		coveredFeatures: [],
+		assemblingFeature: undefined,
 		cellsAssembled: false
 	}
 
@@ -65,8 +66,8 @@ self.body(func(inputs) {
 		}
 	}
 
-	// adding assembling feature to the list of covered features
-	dataDescription.coveredFeatures = append(dataDescription.coveredFeatures, assemblingFeature)
+	// adding assembling feature
+	dataDescription.assemblingFeature = assemblingFeature
 
 	// there should be call join on pfFrames, but it's not implements, so we will do it by hand
 	dataGroupedByDonorId := prepareDonorColumn.groupDataByDonorId(inputs.donorColumn, inputs.datasets)
@@ -75,6 +76,7 @@ self.body(func(inputs) {
 	shmTreeTableOptions := exportSettings.shmTreeTableOptions(dataDescription)
 	shmTreeNodesTableOptions := exportSettings.shmTreeNodesTableOptions(dataDescription)
 	shmTreeNodesWithClonesTableOptions := exportSettings.shmTreeNodesWithClonesTableOptions(dataDescription, inputs.donorColumn)
+	shmTreeNodesUniqueIsotypeTableOptions := exportSettings.shmTreeNodesUniqueIsotypeTableOptions(dataDescription)
 
 	// TODO that call is too low level. Should be replaced with something that works with pColumns, not data only
 	mixcrResults := llPFrames.aggregate(
@@ -94,6 +96,9 @@ self.body(func(inputs) {
 			}, {
 				name: "treeNodesWithClones",
 				type: "Resource"
+			}, {
+				name: "treeNodesUniqueIsotype",
+				type: "Resource"
 			}, {
 				name: "tsvs",
 				type: "Resource"
@@ -120,9 +125,12 @@ self.body(func(inputs) {
 		false,
 		// inputs
 		{
+			shmTreeTableOptions: shmTreeTableOptions,
 			shmTreeTableArgs: shmTreeTableOptions.cmdArgs,
 			shmTreeNodesTableOptions: shmTreeNodesTableOptions,
+			shmTreeNodesWithClonesTableOptions: shmTreeNodesWithClonesTableOptions,
 			shmTreeNodesWithClonesTableArgs: shmTreeNodesWithClonesTableOptions.cmdArgs,
+			shmTreeNodesUniqueIsotypeTableOptions: shmTreeNodesUniqueIsotypeTableOptions,
 			globalParams: maps.merge(
 				inputs.params,
 				{ datasetTypes: datasetTypes }
@@ -158,6 +166,13 @@ self.body(func(inputs) {
         additionalArgsForImportTsv
     )
 
+	treeNodesUniqueIsotype := xsv.importFileMap(
+        mixcrResults.output("treeNodesUniqueIsotype"),
+        "tsv",
+        shmTreeNodesUniqueIsotypeTableOptions.pfconvParams,
+        additionalArgsForImportTsv
+	)
+
 	// Running SOI search for the data
 	soiNodesResults := {}
 	soiTreesResults := {}
@@ -206,6 +221,8 @@ self.body(func(inputs) {
         treeNodes: pframes.exportFrame(treeNodes),
         // combine columns into pFrame
         treeNodesWithClones: pframes.exportFrame(treeNodesWithClones),
+		// combine columns into pFrame
+		treeNodesUniqueIsotype: pframes.exportFrame(treeNodesUniqueIsotype),
 
 		soiNodesResults: maps.mapValues(soiNodesResults, pframes.exportFrame),
 		soiTreesResults: maps.mapValues(soiTreesResults, pframes.exportFrame),

package/src/reconstruct-shm-trees.tpl.tengo

@@ -9,18 +9,21 @@ json := import("json")
 
 // for usage in aggregate function, we should specify all outputs that will be used
 self.defineOutputs(
-    "trees", "treeNodes", "treeNodesWithClones",
+    "trees", "treeNodes", "treeNodesWithClones", "treeNodesUniqueIsotype",
     "tsvs",
     "allelesLog", "treesLog",
     "allelesReport", "treesReport",
     "allelesReportJson", "treesReportJson"
 )
 
+//// import function for aggregating by-nodes output to make it uniquely addressable by it's native key
+tablesAggregation := import(":tablesAggregation")
+
 // import MiXCR as a software to use
 mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:main")
 
 // used to postprocess some tables
-paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-paggregate:main")
+paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
 
 // env for MiXCR to format progress messages
 progressPrefix := "[==PROGRESS==]"
@@ -148,7 +151,7 @@ self.body(func(inputs) {
 
     shmTrees := shmTreesCmdBuilder.run()
     outputShmt := shmTrees.getFile("output.shmt")
-
+    
     // export trees without nodes
     shmTreeExportsCmdBuilder := exec.builder().
         printErrStreamToStdout().
@@ -168,8 +171,9 @@ self.body(func(inputs) {
         saveFile("output.tsv")
 
     shmTreeExports := shmTreeExportsCmdBuilder.run()
-    shmTreeTsv := shmTreeExports.getFile("output.tsv")
+    shmTreeTsvRaw := shmTreeExports.getFile("output.tsv")
 
+    shmTreeTsv := tablesAggregation.ensureUniqueness(shmTreeTsvRaw, inputs.shmTreeTableOptions.pfconvParams, "max_by", "totalReadsCountInTree")
 
     // export tree nodes with data uniq for the node
     shmTreeNodesExportsCmdBuilder := exec.builder().
@@ -192,39 +196,7 @@ self.body(func(inputs) {
     shmTreeNodesExports := shmTreeNodesExportsCmdBuilder.run()
     shmTreeNodesTsvRaw := shmTreeNodesExports.getFile("output.tsv")
 
-    // aggregating by-nodes output to make it uniquely addressable by it's native key
-
-    aggregations := []
-	for col in inputs.shmTreeNodesTableOptions.pfconvParams.columns {
-		aggregations = append(aggregations, {
-			type: "first",
-			src: col.column,
-			dst: col.column
-		})
-	}
-
-    keyColumns := []
-    for axis in inputs.shmTreeNodesTableOptions.pfconvParams.axes {
-		keyColumns = append(keyColumns, axis.column)
-	}
-
-	aggregationWorkflow := { steps: [ {
-			type: "aggregate",
-			groupBy: keyColumns,
-			aggregations: aggregations
-		} ] }
-
-	aggregateCmd := exec.builder().
-        printErrStreamToStdout().
-        software(paggregateSw).
-        arg("--workflow").arg("wf.json").
-		writeFile("wf.json", json.encode(aggregationWorkflow)).
-		arg("input.tsv").addFile("input.tsv", shmTreeNodesTsvRaw).
-		arg("output.tsv").saveFile("output.tsv").
-		run()
-
-	shmTreeNodesTsv := aggregateCmd.getFile("output.tsv")
-
+	shmTreeNodesTsv := tablesAggregation.ensureUniqueness(shmTreeNodesTsvRaw, inputs.shmTreeNodesTableOptions.pfconvParams, "first")
 
     // export nodes with clones. For each node could be several clones
     shmTreeNodesWithClonesExportsCmdBuilder := exec.builder().
@@ -248,17 +220,22 @@ self.body(func(inputs) {
         saveFile("output.tsv")
 
     shmTreeNodesWithClonesExports := shmTreeNodesWithClonesExportsCmdBuilder.run()
-    shmTreeNodesWithClonesTsv := shmTreeNodesWithClonesExports.getFile("output.tsv")
+    shmTreeNodesWithClonesTsvRaw := shmTreeNodesWithClonesExports.getFile("output.tsv")
+
+    shmTreeNodesWithClonesTsv := tablesAggregation.ensureUniqueness(shmTreeNodesWithClonesTsvRaw, inputs.shmTreeNodesWithClonesTableOptions.pfconvParams, "max_by", "readCount")
+    shmTreeNodesUniqueIsotypeTsv := tablesAggregation.ensureUniqueness(shmTreeNodesWithClonesTsv, inputs.shmTreeNodesUniqueIsotypeTableOptions.pfconvParams, "max_by", "readCount")
 
     return {
         trees: shmTreeTsv,
         treeNodes: shmTreeNodesTsv,
         treeNodesWithClones: shmTreeNodesWithClonesTsv,
+        treeNodesUniqueIsotype : shmTreeNodesUniqueIsotypeTsv,
 
         tsvs: {
             trees: shmTreeTsv,
             treeNodes: shmTreeNodesTsv,
-            treeNodesWithClones: shmTreeNodesWithClonesTsv
+            treeNodesWithClones: shmTreeNodesWithClonesTsv,
+            treeNodesUniqueIsotype : shmTreeNodesUniqueIsotypeTsv
         },
 
         allelesLog: alleles.getStdoutStream(),

package/src/soi.tpl.tengo

@@ -12,7 +12,7 @@ self.defineOutputs("nodesResult", "treesResult")
 
 // import MiXCR as a software to use
 mitoolSw := assets.importSoftware("@platforma-open/milaboratories.software-mitool:main")
-paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-paggregate:main")
+paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
 
 inferPartitionKeyLength := func(data) {
 	rType := data.info().Type.Name

package/src/tablesAggregation.lib.tengo

@@ -0,0 +1,75 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+json := import("json")
+paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
+
+//// aggregating by-nodes output to make it uniquely addressable by it's native key
+ensureUniqueness := func(inputTsv, pfConvParams, ...aggParams) {
+    keyColumns := []
+    for axis in pfConvParams.axes {
+        keyColumns = append(keyColumns, axis.column)
+    }
+
+    aggregationWorkflow := undefined
+    if len(aggParams) > 1 {
+        pickCols := []
+        for col in pfConvParams.columns {
+            pickCols = append(pickCols, [
+                col.column,
+                col.column
+                ]
+            )
+        }
+
+        rankingCol := aggParams[1]
+        aggregationWorkflow = { 
+            steps: [
+                {
+                    type: "aggregate",
+                    groupBy: keyColumns,
+                    aggregations: [
+                        {
+                            type: aggParams[0],
+                            rankingCol: rankingCol,
+                            pickCols: pickCols
+                        }           
+                    ]
+                }
+            ] 
+        }
+    } else {
+        aggregations := []
+        for col in pfConvParams.columns {
+            aggregations = append(aggregations, {
+                type: aggParams[0],
+                src: col.column,
+                dst: col.column
+            })
+        }
+
+        aggregationWorkflow = { steps: [ 
+            {
+                type: "aggregate",
+                groupBy: keyColumns,
+                aggregations: aggregations
+                } 
+            ]
+        }
+    }
+
+    aggregateCmd := exec.builder().
+        printErrStreamToStdout().
+        software(paggregateSw).
+        arg("--workflow").arg("wf.json").
+        writeFile("wf.json", json.encode(aggregationWorkflow)).
+        arg("input.tsv").addFile("input.tsv", inputTsv).
+        arg("output.tsv").saveFile("output.tsv").
+        run()
+
+    return aggregateCmd.getFile("output.tsv")
+}
+
+export ll.toStrict({
+	ensureUniqueness: ensureUniqueness
+})