@platforma-open/milaboratories.mixcr-shm-trees.workflow 3.6.1 → 3.8.0

package/src/main.tpl.tengo

@@ -19,9 +19,14 @@ wf.body(func(args) {
 
 	// we could not use array as request for waiting (see below), so we store datasets in a dictionary
 	datasets := {}
+	firstDataset := undefined
 	for datasetRef in args.datasetColumns {
 		// we assume that mixcr block produces exactly one clns column
-		datasets[datasetRef.blockId] = wf.resolve(datasetRef)
+		ds := wf.resolve(datasetRef)
+		datasets[datasetRef.blockId] = ds
+		if is_undefined(firstDataset) {
+			firstDataset = ds
+		}
 	}
 
 	library := render.createEphemeral(requestLibraryTpl, {
@@ -31,13 +36,14 @@ wf.body(func(args) {
 
 	donorColumn := wf.resolve(args.donorColumn)
 
-	// The problem is that refs for data is not resolved.
-	// To deal with it, we should call resolve resolve that will return feature.
-	// Then to resolve feature we should call another templete where we will describe what's to wait
 	results := render.createEphemeral(processTpl, {
 		datasets: datasets,
 		donorColumn: donorColumn,
 		library: library,
+		etc: {
+			firstDatasetSpec: firstDataset.spec,
+			blockId: wf.getBlockId()
+		},
 		params: {
 			downsampling: args.downsampling,
 			sequencesOfInterest: args.sequencesOfInterest
@@ -46,15 +52,18 @@ wf.body(func(args) {
 
 	return {
 		outputs: {
-			trees: results.output("trees"),
-			treeNodes: results.output("treeNodes"),
-			treeNodesWithClones: results.output("treeNodesWithClones"),
-			treeNodesUniqueIsotype: results.output("treeNodesUniqueIsotype"),
+			trees: pframes.exportFrame(results.output("trees")),
+			treeNodes: pframes.exportFrame(results.output("treeNodes")),
+			treeNodesWithClones: pframes.exportFrame(results.output("treeNodesWithClones")),
+			treeNodesUniqueIsotype: pframes.exportFrame(results.output("treeNodesUniqueIsotype")),
+
+			alleles: results.output("alleles"),
+			downsampled: results.output("downsampled"),
+			shmt: results.output("shmt"),
 
 			soiNodesResults: results.output("soiNodesResults"),
 			soiTreesResults: results.output("soiTreesResults"),
 
-			tsvs: results.output("tsvs"),
 			allelesLogs: results.output("allelesLogs"),
 			treesLogs: results.output("treesLogs"),
 

package/src/mixcr-export.tpl.tengo

@@ -0,0 +1,132 @@
+//tengo:hash_override 994D9444-D460-429A-9004-EADA3D49D52E
+
+self := import("@platforma-sdk/workflow-tengo:tpl")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+times := import("times")
+
+json := import("json")
+
+self.defineOutputs(
+    "trees", "treeNodes", "treeNodesWithClones", "treeNodesUniqueIsotype"
+)
+
+tablesAggregation := import(":tables-aggregation")
+
+// import MiXCR as a software to use
+mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:main")
+
+self.body(func(inputs) {
+	shmtFile := inputs[pConstants.VALUE_FIELD_NAME]
+	params := inputs.params
+    library := inputs.library
+    libraryFormat := params.libraryFormat
+    shmTreeArgs := params.shmTreeArgs
+	shmTreeNodesArgs := params.shmTreeNodesArgs
+	shmTreeNodesWithClonesArgs := params.shmTreeNodesWithClonesArgs
+	shmTreeNodesUniqueIsotypeArgs := params.shmTreeNodesUniqueIsotypeArgs
+	shmTreeEnsureUniquenessParams := params.shmTreeEnsureUniquenessParams
+	shmTreeNodesEnsureUniquenessParams := params.shmTreeNodesEnsureUniquenessParams
+	shmTreeNodesWithClonesEnsureUniquenessParams := params.shmTreeNodesWithClonesEnsureUniquenessParams
+	shmTreeNodesUniqueIsotypeEnsureUniquenessParams := params.shmTreeNodesUniqueIsotypeEnsureUniquenessParams
+
+    addLibraryFile := func(cmdBuilder) {
+        if !is_undefined(library) {
+            if libraryFormat == "repseqio.json.gz" {
+                cmdBuilder.addFile("library.json.gz", library)
+            } else {
+                cmdBuilder.addFile("library.json", library)
+            }
+        }
+    }
+
+    // export trees without nodes
+    shmTreeExportsCmdBuilder := exec.builder().
+        printErrStreamToStdout().
+        dontSaveStdoutOrStderr().
+        inLightQueue().
+        secret("MI_LICENSE", "MI_LICENSE").
+        software(mixcrSw).
+        arg("exportShmTrees")
+
+    addLibraryFile(shmTreeExportsCmdBuilder)
+
+    for arg in shmTreeArgs {
+        shmTreeExportsCmdBuilder = shmTreeExportsCmdBuilder.arg(arg)
+    }
+
+    shmTreeExportsCmdBuilder = shmTreeExportsCmdBuilder.
+        arg("input.shmt").
+        arg("output.tsv").
+        addFile("input.shmt", shmtFile).
+        saveFile("output.tsv")
+
+    shmTreeExports := shmTreeExportsCmdBuilder.run()
+    shmTreeTsvRaw := shmTreeExports.getFile("output.tsv")
+
+    shmTreeTsv := tablesAggregation.ensureUniqueness(shmTreeTsvRaw, shmTreeEnsureUniquenessParams, "max_by", "totalReadsCountInTree")
+
+    // export tree nodes with data uniq for the node
+    shmTreeNodesExportsCmdBuilder := exec.builder().
+        printErrStreamToStdout().
+        dontSaveStdoutOrStderr().
+        inLightQueue().
+        secret("MI_LICENSE", "MI_LICENSE").
+        software(mixcrSw).
+        arg("exportShmTreesWithNodes")
+
+    addLibraryFile(shmTreeNodesExportsCmdBuilder)
+
+    for arg in shmTreeNodesArgs {
+        shmTreeNodesExportsCmdBuilder = shmTreeNodesExportsCmdBuilder.arg(arg)
+    }
+
+    shmTreeNodesExportsCmdBuilder = shmTreeNodesExportsCmdBuilder.
+        arg("input.shmt").
+        arg("output.tsv").
+        addFile("input.shmt", shmtFile).
+        saveFile("output.tsv")
+
+    shmTreeNodesExports := shmTreeNodesExportsCmdBuilder.run()
+    shmTreeNodesTsvRaw := shmTreeNodesExports.getFile("output.tsv")
+
+	shmTreeNodesTsv := tablesAggregation.ensureUniqueness(shmTreeNodesTsvRaw, shmTreeNodesEnsureUniquenessParams, "first")
+
+    // export nodes with clones. For each node could be several clones
+    shmTreeNodesWithClonesExportsCmdBuilder := exec.builder().
+        printErrStreamToStdout().
+        dontSaveStdoutOrStderr().
+        inLightQueue().
+		secret("MI_LICENSE", "MI_LICENSE").
+		software(mixcrSw).
+        arg("exportShmTreesWithNodes").
+        // don't export nodes that don't have clones
+        arg("--only-observed")
+
+    addLibraryFile(shmTreeNodesWithClonesExportsCmdBuilder)
+
+    for arg in shmTreeNodesWithClonesArgs {
+        shmTreeNodesWithClonesExportsCmdBuilder = shmTreeNodesWithClonesExportsCmdBuilder.arg(arg)
+    }
+
+    shmTreeNodesWithClonesExportsCmdBuilder = shmTreeNodesWithClonesExportsCmdBuilder.
+        arg("input.shmt").
+        arg("output.tsv").
+        addFile("input.shmt", shmtFile).
+        saveFile("output.tsv")
+
+    shmTreeNodesWithClonesExports := shmTreeNodesWithClonesExportsCmdBuilder.run()
+    shmTreeNodesWithClonesTsvRaw := shmTreeNodesWithClonesExports.getFile("output.tsv")
+
+    shmTreeNodesWithClonesTsv := tablesAggregation.ensureUniqueness(shmTreeNodesWithClonesTsvRaw, shmTreeNodesWithClonesEnsureUniquenessParams, "max_by", "readCount")
+    shmTreeNodesUniqueIsotypeTsv := tablesAggregation.ensureUniqueness(shmTreeNodesWithClonesTsv, shmTreeNodesUniqueIsotypeEnsureUniquenessParams, "max_by", "readCount")
+
+    return {
+        trees: shmTreeTsv,
+        treeNodes: shmTreeNodesTsv,
+        treeNodesWithClones: shmTreeNodesWithClonesTsv,
+        treeNodesUniqueIsotype : shmTreeNodesUniqueIsotypeTsv
+    }
+})

package/src/{reconstruct-shm-trees.tpl.tengo → mixcr-shm-trees.tpl.tengo}

@@ -1,30 +1,27 @@
+//tengo:hash_override 33125CD9-6F67-46B3-AA68-0FCD825B51BA
+
 self := import("@platforma-sdk/workflow-tengo:tpl")
 pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 exec := import("@platforma-sdk/workflow-tengo:exec")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+maps := import("@platforma-sdk/workflow-tengo:maps")
 times := import("times")
 
 json := import("json")
 
 // for usage in aggregate function, we should specify all outputs that will be used
 self.defineOutputs(
-    "trees", "treeNodes", "treeNodesWithClones", "treeNodesUniqueIsotype",
-    "tsvs",
+    "alleles", "downsampled", "shmt",
     "allelesLog", "treesLog",
     "allelesReport", "treesReport",
     "allelesReportJson", "treesReportJson"
 )
 
-//// import function for aggregating by-nodes output to make it uniquely addressable by it's native key
-tablesAggregation := import(":tablesAggregation")
-
 // import MiXCR as a software to use
 mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:main")
 
-// used to postprocess some tables
-paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
-
 // env for MiXCR to format progress messages
 progressPrefix := "[==PROGRESS==]"
 
@@ -32,9 +29,9 @@ self.body(func(inputs) {
 	inputData := inputs[pConstants.VALUE_FIELD_NAME]
     globalParams := inputs.globalParams
     datasetTypes := globalParams.datasetTypes
-    downsampling := globalParams.downsampling
     library := inputs.library
     libraryFormat := globalParams.libraryFormat
+    downsampling := globalParams.downsampling
 
     ll.assert(!is_undefined(datasetTypes), "datasetTypes undefined")
 
@@ -50,6 +47,10 @@ self.body(func(inputs) {
 
     // seed := times.time_string(times.now())
 
+    //
+    // Alleles inference
+    //
+
     allelesCmdBuilder := exec.builder().
         printErrStreamToStdout().
         secret("MI_LICENSE", "MI_LICENSE").
@@ -70,14 +71,14 @@ self.body(func(inputs) {
         allelesCmdBuilder.env("SEED", globalParams.seed)
     }
 
-    toProcess := []
+    entries := []
     for sKey, inputFile in inputData.inputs() {
         key := json.decode(sKey)
         sampleId := key[0]
         clonotypingBlockId := key[1]
         // file name should encode axis values. It will be parsed by xsv.importFileMap afterwards to restore axis for clones data
         fileName := sampleId + "___" + clonotypingBlockId + ".clns"
-        toProcess = append(toProcess, {
+        entries = append(entries, {
             clonotypingBlockId: clonotypingBlockId,
             sampleId: sampleId,
             fileName: fileName,
@@ -85,18 +86,23 @@ self.body(func(inputs) {
         })
     }
 
-    for input in toProcess {
-        allelesCmdBuilder.addFile(input.fileName, input.input).
-            arg(input.fileName).
-            saveFile("alleles/" + input.fileName)
+    for entry in entries {
+        allelesCmdBuilder.addFile(entry.fileName, entry.input).
+            arg(entry.fileName).
+            saveFile("alleles/" + entry.fileName)
     }
 
     alleles := allelesCmdBuilder.run()
 
-    for input in toProcess {
-        input.alleles = alleles.getFile("alleles/" + input.fileName)
+    for entry in entries {
+        entry.afterAlleles = alleles.getFile("alleles/" + entry.fileName)
+        entry.treesInput = entry.afterAlleles
     }
 
+    //
+    // Optional downsampling
+    //
+
     if !is_undefined(downsampling) {
         downsamplingParam := ""
         if downsampling.type == "CountReadsFixed" {
@@ -115,8 +121,8 @@ self.body(func(inputs) {
             ll.panic("Unknown downsampling type: " + downsampling.type)
         }
 
-        for input in toProcess {
-            if datasetTypes[input.clonotypingBlockId] == "bulk" {
+        for entry in entries {
+            if datasetTypes[entry.clonotypingBlockId] == "bulk" {
                 downsamplingCmdBuilder := exec.builder().
                     printErrStreamToStdout().
                     secret("MI_LICENSE", "MI_LICENSE").
@@ -126,15 +132,21 @@ self.body(func(inputs) {
                     arg("--downsampling").
                     arg(downsamplingParam).
                     arg("clones.clns").
-                    addFile("clones.clns", input.alleles).
+                    addFile("clones.clns", entry.afterAlleles).
                     saveFile("clones.downsampled.clns")
                 addLibraryFile(downsamplingCmdBuilder)
                 downsamplingCmd := downsamplingCmdBuilder.run()
-                input.alleles = downsamplingCmd.getFile("clones.downsampled.clns")
+
+                // overriding trees input with downsampled file
+                entry.treesInput = downsamplingCmd.getFile("clones.downsampled.clns")
             }
         }
     }
 
+    //
+    // SHM trees inference
+    //
+
     shmTreesCmdBuilder := exec.builder().
         printErrStreamToStdout().
         secret("MI_LICENSE", "MI_LICENSE").
@@ -152,10 +164,10 @@ self.body(func(inputs) {
 
     addLibraryFile(shmTreesCmdBuilder)
 
-    for input in toProcess {
+    for entry in entries {
         shmTreesCmdBuilder.
-            addFile(input.fileName, input.alleles).
-            arg(input.fileName)
+            addFile(entry.fileName, entry.treesInput).
+            arg(entry.fileName)
     }
 
     shmTreesCmdBuilder.arg("output.shmt").saveFile("output.shmt")
@@ -163,107 +175,29 @@ self.body(func(inputs) {
     shmTrees := shmTreesCmdBuilder.run()
     outputShmt := shmTrees.getFile("output.shmt")
 
-    // export trees without nodes
-    shmTreeExportsCmdBuilder := exec.builder().
-        printErrStreamToStdout().
-        dontSaveStdoutOrStderr().
-        inLightQueue().
-        secret("MI_LICENSE", "MI_LICENSE").
-        software(mixcrSw).
-        arg("exportShmTrees")
-
-    addLibraryFile(shmTreeExportsCmdBuilder)
-
-    for arg in inputs.shmTreeTableArgs {
-        shmTreeExportsCmdBuilder = shmTreeExportsCmdBuilder.arg(arg)
-    }
-
-    shmTreeExportsCmdBuilder = shmTreeExportsCmdBuilder.
-        arg("input.shmt").
-        arg("output.tsv").
-        addFile("input.shmt", outputShmt).
-        saveFile("output.tsv")
-
-    shmTreeExports := shmTreeExportsCmdBuilder.run()
-    shmTreeTsvRaw := shmTreeExports.getFile("output.tsv")
-
-    shmTreeTsv := tablesAggregation.ensureUniqueness(shmTreeTsvRaw, inputs.shmTreeTableOptions.pfconvParams, "max_by", "totalReadsCountInTree")
-
-    // export tree nodes with data uniq for the node
-    shmTreeNodesExportsCmdBuilder := exec.builder().
-        printErrStreamToStdout().
-        dontSaveStdoutOrStderr().
-        inLightQueue().
-        secret("MI_LICENSE", "MI_LICENSE").
-        software(mixcrSw).
-        arg("exportShmTreesWithNodes")
-
-    addLibraryFile(shmTreeNodesExportsCmdBuilder)
-
-    for arg in inputs.shmTreeNodesTableOptions.cmdArgs {
-        shmTreeNodesExportsCmdBuilder = shmTreeNodesExportsCmdBuilder.arg(arg)
-    }
-
-    shmTreeNodesExportsCmdBuilder = shmTreeNodesExportsCmdBuilder.
-        arg("input.shmt").
-        arg("output.tsv").
-        addFile("input.shmt", outputShmt).
-        saveFile("output.tsv")
-
-    shmTreeNodesExports := shmTreeNodesExportsCmdBuilder.run()
-    shmTreeNodesTsvRaw := shmTreeNodesExports.getFile("output.tsv")
-
-	shmTreeNodesTsv := tablesAggregation.ensureUniqueness(shmTreeNodesTsvRaw, inputs.shmTreeNodesTableOptions.pfconvParams, "first")
-
-    // export nodes with clones. For each node could be several clones
-    shmTreeNodesWithClonesExportsCmdBuilder := exec.builder().
-        printErrStreamToStdout().
-        dontSaveStdoutOrStderr().
-        inLightQueue().
-        secret("MI_LICENSE", "MI_LICENSE").
-        env("MI_PROGRESS_PREFIX", progressPrefix).
-        software(mixcrSw).
-        arg("exportShmTreesWithNodes").
-        // don't export nodes that don't have clones
-        arg("--only-observed")
-
-    addLibraryFile(shmTreeNodesWithClonesExportsCmdBuilder)
-
-    for arg in inputs.shmTreeNodesWithClonesTableArgs {
-        shmTreeNodesWithClonesExportsCmdBuilder = shmTreeNodesWithClonesExportsCmdBuilder.arg(arg)
-    }
-
-    shmTreeNodesWithClonesExportsCmdBuilder = shmTreeNodesWithClonesExportsCmdBuilder.
-        arg("input.shmt").
-        arg("output.tsv").
-        addFile("input.shmt", outputShmt).
-        saveFile("output.tsv")
-
-    shmTreeNodesWithClonesExports := shmTreeNodesWithClonesExportsCmdBuilder.run()
-    shmTreeNodesWithClonesTsvRaw := shmTreeNodesWithClonesExports.getFile("output.tsv")
+    return {
+        // returning to be cached
+        alleles: smart.createMapResource(maps.mapValues(entries, func(entry) {
+            return entry.afterAlleles
+        })),
 
-    shmTreeNodesWithClonesTsv := tablesAggregation.ensureUniqueness(shmTreeNodesWithClonesTsvRaw, inputs.shmTreeNodesWithClonesTableOptions.pfconvParams, "max_by", "readCount")
-    shmTreeNodesUniqueIsotypeTsv := tablesAggregation.ensureUniqueness(shmTreeNodesWithClonesTsv, inputs.shmTreeNodesUniqueIsotypeTableOptions.pfconvParams, "max_by", "readCount")
+        // returning to be cached
+        downsampled: smart.createMapResource(maps.mapValues(entries, func(entry) {
+            return entry.treesInput
+        })),
 
-    return {
-        trees: shmTreeTsv,
-        treeNodes: shmTreeNodesTsv,
-        treeNodesWithClones: shmTreeNodesWithClonesTsv,
-        treeNodesUniqueIsotype : shmTreeNodesUniqueIsotypeTsv,
-
-        tsvs: {
-            trees: shmTreeTsv,
-            treeNodes: shmTreeNodesTsv,
-            treeNodesWithClones: shmTreeNodesWithClonesTsv,
-            treeNodesUniqueIsotype : shmTreeNodesUniqueIsotypeTsv
-        },
+        // main output
+        shmt: outputShmt,
 
+        // logs
         allelesLog: alleles.getStdoutStream(),
         treesLog: shmTrees.getStdoutStream(),
 
+        // reports
         allelesReport: alleles.getFile("report.txt"),
         treesReport: shmTrees.getFile("report.txt"),
 
+        // json reports
         allelesReportJson: alleles.getFile("report.json"),
         treesReportJson: shmTrees.getFile("report.json")
     }

package/src/prepare-donor-column.lib.tengo

@@ -1,16 +1,18 @@
 ll := import("@platforma-sdk/workflow-tengo:ll")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 maps := import("@platforma-sdk/workflow-tengo:maps")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
 
 json := import("json")
 
 _P_COLUMN_DATA_RESOURCE_MAP := { Name: "PColumnData/ResourceMap", Version: "1" }
 
-groupDataByDonorId := func(donorColumn, datasets) {
+groupDataByDonorId := func(donorColumn, datasets, firstDatasetSpec) {
     // we need to form a pColumn with two axes:
 	//    axes[0]: donorId
 	//    axes[1]: sampleId
-	//    axes[2]: mixcrclonotypingBlockId
+	//    axes[2]: mixcrClonotypingBlockId
 	//      value: fileRef resource
 
 	// we have:
@@ -23,9 +25,11 @@ groupDataByDonorId := func(donorColumn, datasets) {
 
     donorColumnSpec := donorColumn.spec
 
+    trace := pSpec.makeTrace(firstDatasetSpec)
+
     sampleIdAxis := donorColumnSpec.axesSpec[0]
 
-    resultSpec := maps.clone({
+    resultSpec := trace.inject(maps.clone({
         kind: "PColumn",
         name: "mixcr.com/clns",
         valueType: "File",
@@ -47,7 +51,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
                 }
             }
         ]
-    }, { removeUndefs: true })
+    }, { removeUndefs: true }))
 
     // creating sample to donor map