@platforma-open/milaboratories.mixcr-clonotyping-2.workflow 2.2.2 → 2.3.0

package/src/calculate-export-specs.lib.tengo

@@ -79,6 +79,12 @@ exportSpecOpsFromPreset := func(presetSpecForBack) {
 	}
 }
 
+addSpec := func(columns, additionalSpec) {
+	return slices.map(columns, func(columnSpec) {
+		return maps.deepMerge(columnSpec, additionalSpec)
+	})
+}
+
 calculateExportSpecs := func(presetSpecForBack, blockId) {
 	ops := exportSpecOpsFromPreset(presetSpecForBack)
 
@@ -87,19 +93,30 @@ calculateExportSpecs := func(presetSpecForBack, blockId) {
 	hasUmi := ops.hasUmi
 	splitByC := ops.splitByC
 
+	isSingleCell := !is_undefined(cellTags) && len(cellTags) > 0
+
 	assemblingFeatureInfo := assemblingFeatureInfo(assemblingFeature)
 	productiveFeature := assemblingFeatureInfo.productiveFeature
 	coreGeneFeatures := assemblingFeatureInfo.coreGeneFeatures
 
 	clonotypeKeyColumns := undefined
-	if !is_undefined(assemblingFeature) && (is_undefined(cellTags) || len(cellTags) == 0) {
+	clonotypeKeyArgs := undefined
+	if !is_undefined(assemblingFeature) {
 		clonotypeKeyColumns = ["nSeq" + assemblingFeature, "bestVGene", "bestJGene"]
+		clonotypeKeyArgs = [
+			[ "-nFeature", assemblingFeature ],
+			[ "-vGene" ],
+			[ "-jGene" ]
+		]
 		if splitByC {
 			clonotypeKeyColumns += ["bestCGene"]
+			clonotypeKeyArgs += [ [ "-cGene" ] ]
 		}
+
 	}
 
 	columnsSpecPerSample := []
+	columnsSpecPerSampleSc := undefined
 	columnsSpecPerClonotype := []
 
 	// array of array of arg groups
@@ -191,6 +208,52 @@ calculateExportSpecs := func(presetSpecForBack, blockId) {
 		mainAbundanceColumn = "uniqueMoleculeFraction"
 	}
 
+	if isSingleCell {
+		// copying reads and umi counts and fraction removing isPrimary and isAnchor
+		// addSpec(columnsSpecPerSample, {
+		// 	spec: {
+		// 		annotations: {
+		// 			"pl7.app/abundance/isPrimary": undefined,
+		// 			"pl7.app/isAnchor": undefined
+		// 		}
+		// 	}
+		// }) +
+		columnsSpecPerSampleSc = [ {
+			column: "uniqueCellCount",
+			id: "cell-count",
+			allowNA: false,
+			spec: {
+				name: "pl7.app/vdj/uniqueCellCount",
+				valueType: "Long",
+				annotations: a(88000, true, {
+					"pl7.app/min": "1",
+					"pl7.app/isAbundance": "true",
+					"pl7.app/abundance/unit": "cells",
+					"pl7.app/abundance/normalized": "false",
+					"pl7.app/abundance/isPrimary": "true",
+					"pl7.app/isAnchor": "true",
+					"pl7.app/label": "Number of Cells"
+				})
+			}
+		}, {
+			column: "uniqueCellFraction",
+			id: "cell-fraction",
+			allowNA: false,
+			spec: {
+				name: "pl7.app/vdj/uniqueCellFraction",
+				valueType: "Double",
+				annotations: a(88000, true, {
+					"pl7.app/min": "0",
+					"pl7.app/isAbundance": "true",
+					"pl7.app/abundance/unit": "cells",
+					"pl7.app/abundance/normalized": "true",
+					"pl7.app/abundance/isPrimary": "true",
+					"pl7.app/label": "Fraction of Cells"
+				})
+			}
+		} ]
+	}
+
 	// VDJC Hits
 
 	orderP := 80000
@@ -310,21 +373,21 @@ calculateExportSpecs := func(presetSpecForBack, blockId) {
 	orderP = 10000
 
 	mutationColumnVariants := [ {
-			name: "Mutations",
-			valueType: "String",
-			labelPart: " mutations in ",
-			idPart: "-mutations-"
-		}, {
-			name: "MutationsCount",
-			valueType: "Int",
-			labelPart: " mutations count in ",
-			idPart: "-mutations-count-"
-		}, {
-			name: "MutationsRate",
-			valueType: "Double",
-			labelPart: " mutations rate in ",
-			idPart: "-mutations-rate-"
-		} ]
+		name: "Mutations",
+		valueType: "String",
+		labelPart: " mutations in ",
+		idPart: "-mutations-"
+	}, {
+		name: "MutationsCount",
+		valueType: "Int",
+		labelPart: " mutations count in ",
+		idPart: "-mutations-count-"
+	}, {
+		name: "MutationsRate",
+		valueType: "Double",
+		labelPart: " mutations rate in ",
+		idPart: "-mutations-rate-"
+	} ]
 
 	for isAminoAcid in [false, true] {
 		alphabetShort := isAminoAcid ? "AA" : "Nt"
@@ -465,12 +528,6 @@ calculateExportSpecs := func(presetSpecForBack, blockId) {
 
 	// Adding block and segment annotations
 
-	addSpec := func(columns, additionalSpec) {
-		return slices.map(columns, func(columnSpec) {
-			return maps.deepMerge(columnSpec, additionalSpec)
-		})
-	}
-
 	columnsSpecPerSample = addSpec(columnsSpecPerSample, { spec: { domain: {
 		"pl7.app/vdj/clonotypingRunId": blockId
 	} } })
@@ -500,10 +557,10 @@ calculateExportSpecs := func(presetSpecForBack, blockId) {
 	// Axes
 
 	axesByClonotypeKey := undefined
+	axesByScClonotypeKey := undefined
+	cellTagColumns := undefined
 
 	if !is_undefined(clonotypeKeyColumns) {
-		ll.assert(is_undefined(cellTags) || len(cellTags) == 0, "cellTags and clonotypeKeyAxes cannot both be defined")
-
 		// checking that corresponding columns exist in export
 		keyStrincture := []
 		for keyColumn in clonotypeKeyColumns {
@@ -515,93 +572,70 @@ calculateExportSpecs := func(presetSpecForBack, blockId) {
 		}
 
 		axesByClonotypeKey = [ {
-				column: "clonotypeKey",
+			column: "clonotypeKey",
+			naRegex: "",
+			spec: {
+				name: "pl7.app/vdj/clonotypeKey",
+				type: "String",
+				domain: {
+					"pl7.app/vdj/clonotypeKey/structure": string(json.encode(keyStrincture))
+				},
+				annotations: {
+					"pl7.app/label": "Clonotype key",
+					"pl7.app/table/visibility": "optional",
+					"pl7.app/table/orderPriority": "110000"
+				}
+			}
+		} ]
+
+		if isSingleCell {
+			cellTagColumns = slices.map(cellTags, func(cellTag) {
+				return "tagValue" + cellTag
+			})
+
+			axesByScClonotypeKey = [ {
+				column: "scClonotypeKey",
 				naRegex: "",
 				spec: {
-					name: "pl7.app/vdj/clonotypeKey",
+					name: "pl7.app/vdj/scClonotypeKey",
 					type: "String",
 					domain: {
-						"pl7.app/vdj/clonotypeKey/structure": string(json.encode(keyStrincture))
+						"pl7.app/vdj/scClonotypeKey/structure": string(json.encode(keyStrincture))
 					},
 					annotations: {
-						"pl7.app/label": "Clonotype key",
+						"pl7.app/label": "SC Clonotype key",
 						"pl7.app/table/visibility": "optional",
 						"pl7.app/table/orderPriority": "110000"
 					}
 				}
 			} ]
-	}
 
-	axesByClonotypeId := [ {
-		column: "cloneId",
-		spec: {
-			name: "pl7.app/vdj/cloneId",
-			type: "Long",
-			domain: {
-				"pl7.app/blockId": blockId
-			},
-			annotations: {
-				"pl7.app/min": "0",
-				"pl7.app/label": "Clone id",
-				"pl7.app/table/visibility": "optional",
-				"pl7.app/table/orderPriority": "90000"
-				}
-			}
-		} ]
-	exportArgs += [ [ "-cloneId" ] ]
-
-	orderP = 100000
-	if !is_undefined(cellTags) && len(cellTags) > 0 {
-		for tag in cellTags {
-			label := undefined
-			if tag == "CELL" {
-				label = "Cell tag"
-			} else {
-				label = text.to_title(tag[:4]) + " " + text.to_lower(tag[4:])
-			}
-			axesByClonotypeId += [ {
-					column: "tagValue" + tag,
-					naRegex: "",
-					spec: {
-						name: "pl7.app/vdj/cellTag",
-						type: "String",
-						domain: {
-							"pl7.app/vdj/cellTagId": tag,
-							"pl7.app/vdj/clonotypingRunId": blockId
-						},
-						annotations: a(orderP, true, {
-							"pl7.app/label": label
-						})
-					}
-				} ]
-			orderP -= 1
+			//	exportArgs += [ [ "-tags", "Cell" ] ]
+			// 	axesByClonotypeKeyAndCellTag = axesByClonotypeKey + [ {
+			// 		column: "cellTag",
+			// 		naRegex: "",
+			// 		spec: {
+			// 			name: "pl7.app/vdj/cellTag",
+			// 			type: "String",
+			// 			annotations: a(orderP, true, {
+			// 				"pl7.app/label": "Cell tag"
+			// 			})
+			// 		}
+			// 	} ]
 		}
-		exportArgs += [ [ "-tags", "Cell" ] ]
-
-		columnsSpec += [ {
-				column: "cellGroup",
-				id: "cell-group",
-				naRegex: "undefined|contamination",
-				allowNA: true,
-				spec: {
-					name: "pl7.app/vdj/cellGroup",
-					valueType: "Long",
-					annotations: a(24000, true, {
-						"pl7.app/min": "0",
-						"pl7.app/label": "Cell group number"
-					} )
-				}
-			} ]
-		exportArgs += [ [ "-cellGroup" ] ]
 	}
 
 	return {
 		clonotypeKeyColumns: clonotypeKeyColumns,
+		clonotypeKeyArgs: clonotypeKeyArgs,
+
+		cellTagColumns: cellTagColumns,
 
-		axesByClonotypeId: axesByClonotypeId,
 		axesByClonotypeKey: axesByClonotypeKey,
+		axesByScClonotypeKey: axesByScClonotypeKey,
 
 		columnsSpecPerSample: columnsSpecPerSample,
+		columnsSpecPerSampleSc: columnsSpecPerSampleSc,
 		columnsSpecPerClonotype: columnsSpecPerClonotype,
 
 		columnsSpec: columnsSpec,

package/src/main.tpl.tengo

@@ -93,9 +93,11 @@ wf.body(func(args) {
 	presetSpecForBack := presetInfoResult.output("presetSpecForBack", 24 * 60 * 60 * 1000)
 
 	// calculating chains
-	chains := ["IGHeavy", "IGLight", "TRAlpha", "TRBeta", "TRGamma", "TRDelta"]
+	// chains := ["IGHeavy", "IGLight", "TRAlpha", "TRBeta", "TRGamma", "TRDelta"]
+	//receptorsOrChains := ["IG", "TCRAB", "TCRGD"]
+ 	receptorsOrChains := ["IG"]
 	if !is_undefined(args.chains) {
-		chains = args.chains
+		receptorsOrChains = args.chains
 	}
 
 	runMixcr := render.createEphemeral(processTpl, {
@@ -110,7 +112,7 @@ wf.body(func(args) {
 
 		params: smart.createJsonResource({
 			species: species,
-			chains: chains,
+			receptorsOrChains: receptorsOrChains,
 			limitInput: limitInput,
 			blockId: blockId,
 			presetCommonName: args.presetCommonName,
@@ -127,7 +129,7 @@ wf.body(func(args) {
 			spec: runMixcr.output("reports.spec"),
 			data: runMixcr.output("reports.data")
 		},
-		clones: runMixcr.output("clonotypes"),
+		clonotypes: runMixcr.output("clonotypes"),
 		clns: {
 			spec: runMixcr.output("clns.spec"),
 			data: runMixcr.output("clns.data")

package/src/mixcr-analyze.tpl.tengo

@@ -37,7 +37,7 @@ self.body(func(inputs) {
 
 	presetContent := inputs.presetContent.getDataAsJson()
 
-	if !is_map(presetContent) {
+	if !is_map(presetContent) || !is_array(presetContent.pipeline) {
 		ll.panic("malformed presetContent %v", presetContent)
 	}
 
@@ -45,7 +45,7 @@ self.body(func(inputs) {
 
 	hasAssembleContigs := false
 	hasAssembleCells := false
-	for stage in presetContent.analysisStages {
+	for stage in presetContent.pipeline {
 		if stage == "assembleContigs" {
 			hasAssembleContigs = true
 		} else if stage == "assembleCells" {

package/src/mixcr-export.tpl.tengo

@@ -1,12 +1,13 @@
 ll := import("@platforma-sdk/workflow-tengo:ll")
 self := import("@platforma-sdk/workflow-tengo:tpl")
 pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+smart := import("@platforma-sdk/workflow-tengo:smart")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 exec := import("@platforma-sdk/workflow-tengo:exec")
 
 json := import("json")
 
-self.defineOutputs("tsv")
+self.defineOutputs("tsv", "tsvForSingleCell")
 
 mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:low-memory")
 ptransformSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
@@ -19,37 +20,49 @@ self.body(func(inputs) {
 	exportArgs := params.exportArgs
 
 	clonotypeKeyColumns := params.clonotypeKeyColumns
+	clonotypeKeyArgs := params.clonotypeKeyArgs
+	cellTagColumns := params.cellTagColumns
 
 	// Exporting clones from clns file
 
-	mixcrCmdBuilder := exec.builder().
-		inMediumQueue().
-		printErrStreamToStdout().
-		software(mixcrSw).
-		secret("MI_LICENSE", "MI_LICENSE").
-		arg("exportClones").
-		arg("--dont-split-files").
-		arg("--chains").arg(chains)
-
-	for argGrp in exportArgs {
-		for arg in argGrp {
-			mixcrCmdBuilder.arg(arg)
-		}
+	createExport := func(additionalAction) {
+		mixcrCmdBuilder := exec.builder().
+			inMediumQueue().
+			printErrStreamToStdout().
+			software(mixcrSw).
+			secret("MI_LICENSE", "MI_LICENSE").
+			arg("exportClones").
+			arg("--dont-split-files").
+			arg("--drop-default-fields").
+			arg("--reset-export-clone-table-splitting").
+			arg("--chains").arg(chains)
+
+		additionalAction(mixcrCmdBuilder)
+
+		return mixcrCmdBuilder.
+			arg("clones.clns").
+			addFile("clones.clns", clnsFile).
+			arg("clones.tsv").
+			saveFile("clones.tsv").
+			run()
 	}
 
-	mixcrCmd := mixcrCmdBuilder.
-		arg("clones.clns").
-		addFile("clones.clns", clnsFile).
-		arg("clones.tsv").
-		saveFile("clones.tsv").
-		run()
+	mixcrCmd := createExport(func(mixcrCmdBuilder) {
+		for argGrp in exportArgs {
+			for arg in argGrp {
+				mixcrCmdBuilder.arg(arg)
+			}
+		}
+	})
 
 	unprocessedTsv := mixcrCmd.getFile("clones.tsv")
 
+	result := {
+		tsvForSingleCell: smart.createNullResource()
+	}
+
 	if is_undefined(clonotypeKeyColumns) {
-		return {
-			tsv: unprocessedTsv
-		}
+		result.tsv = unprocessedTsv
 	} else {
 		// Adding clonotypeKey column
 		pWorkflow := {
@@ -71,8 +84,53 @@ self.body(func(inputs) {
 
 		processedTsv := aggregateCmd.getFile("output.tsv")
 
-		return {
-			tsv: processedTsv
+		result.tsv = processedTsv
+	}
+
+	if !is_undefined(cellTagColumns) {
+		mixcrForSingleCell := createExport(func(mixcrCmdBuilder) {
+			mixcrCmdBuilder.
+				arg("--split-by-tags").arg("Cell").
+				arg("-tags").arg("Cell").
+				arg("-readCount").
+				arg("-isProductive").arg("CDR3")
+
+			for argGrp in clonotypeKeyArgs {
+				for arg in argGrp {
+					mixcrCmdBuilder.arg(arg)
+				}
+			}
+		})
+
+		if is_undefined(clonotypeKeyColumns) {
+			ll.panic("clonotypeKeyColumns is undefined")
 		}
+
+		unprocessedTsvForSingleCell := mixcrForSingleCell.getFile("clones.tsv")
+
+		pWorkflow := {
+			steps: [ {
+				type: "combine_columns_as_json",
+				src: clonotypeKeyColumns,
+				dst: "clonotypeKey"
+			}, {
+				type: "combine_columns_as_json",
+				src: cellTagColumns,
+				dst: "cellTag"
+			} ]
+		}
+
+		aggregateCmd := exec.builder().
+			printErrStreamToStdout().
+			software(ptransformSw).
+			arg("--workflow").arg("wf.json").
+			writeFile("wf.json", json.encode(pWorkflow)).
+			arg("input.tsv").addFile("input.tsv", unprocessedTsvForSingleCell).
+			arg("output.tsv").saveFile("output.tsv").
+			run()
+
+		result.tsvForSingleCell = aggregateCmd.getFile("output.tsv")
 	}
+
+	return result
 })

package/src/process-single-cell.tpl.tengo

@@ -0,0 +1,108 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+
+json := import("json")
+
+self.defineOutputs("abundanceTsv", "propertiesAPrimaryTsv", "propertiesASecondaryTsv", "propertiesBPrimaryTsv", "propertiesBSecondaryTsv")
+
+scGroupBuilderSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-clonotyping-2.single-cell-scripts:sc-group-builder")
+scPreprocessingSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-clonotyping-2.single-cell-scripts:preprocessing")
+scOutputProcessingSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-clonotyping-2.single-cell-scripts:output-processing")
+
+self.body(func(inputs) {
+	byCellTagA := inputs[pConstants.VALUE_FIELD_NAME]
+	inputDataMeta := byCellTagA.getDataAsJson()
+	ll.assert(inputDataMeta.keyLength == 1, "unexpected number of aggregation axes")
+
+	byCellTagB := inputs.byCellTagB
+	propertiesA := inputs.propertiesA
+	propertiesB := inputs.propertiesB
+
+	// Creating files map
+	filesMap := {}
+	for k, v in byCellTagA.inputs() {
+		key := json.decode(k)
+		fileName := "by_cell_a_" + key[0] + ".tsv"
+		filesMap[fileName] = v
+	}
+
+	for k, v in byCellTagB.inputs() {
+		key := json.decode(k)
+		fileName := "by_cell_b_" + key[0] + ".tsv"
+		filesMap[fileName] = v
+	}
+
+	scPreprocessingCmd := exec.builder().
+		printErrStreamToStdout().
+		software(scPreprocessingSw).
+		addFiles(filesMap)
+
+	for name, f in filesMap {
+		scPreprocessingCmd = scPreprocessingCmd.arg(name)
+	}
+
+	// Data preprocessing
+	scPreprocessingCmd = scPreprocessingCmd.arg("--output_chainA").arg("chain_a_output.tsv").
+		arg("--output_chainB").arg("chain_b_output.tsv").
+		saveFile("chain_a_output.tsv").
+		saveFile("chain_b_output.tsv")
+	run := scPreprocessingCmd.run()
+	chainAoutput := run.getFile("chain_a_output.tsv")
+	chainBoutput := run.getFile("chain_b_output.tsv")
+
+
+	// Generating scClonotypeKey and abundanceTable
+	scClonotypeBuilderCmd := exec.builder().
+		printErrStreamToStdout().
+		software(scGroupBuilderSw).
+		addFile("chain_a_output.tsv", chainAoutput).
+		addFile("chain_b_output.tsv", chainBoutput).
+		arg("--chainA").arg("chain_a_output.tsv").
+		arg("--chainB").arg("chain_b_output.tsv").
+		arg("--output_clonotype").arg("clonotype.tsv").
+		arg("--output_cell").arg("abundance.tsv").
+		saveFile("clonotype.tsv").
+		saveFile("abundance.tsv")
+	scClonotypeCmd := scClonotypeBuilderCmd.run()
+
+	clonotypeTsv := scClonotypeCmd.getFile("clonotype.tsv")
+	abundanceTsv := scClonotypeCmd.getFile("abundance.tsv")
+
+	propertiesAFile := propertiesA.inputs()["[]"]
+	propertiesBFile := propertiesB.inputs()["[]"]
+
+	// Propagate scClonotypeKey to properties tables
+	scOutputProcessingBuilderCmd := exec.builder().
+		printErrStreamToStdout().
+		software(scOutputProcessingSw).
+		addFile("clonotype.tsv", clonotypeTsv).
+		arg("--main_table").arg("clonotype.tsv").
+		addFile("properties_a.tsv", propertiesAFile).
+		addFile("properties_b.tsv", propertiesBFile).
+		arg("--properties_a").arg("properties_a.tsv").
+		arg("--properties_b").arg("properties_b.tsv").
+		arg("--output_A1").arg("properties_a_primary.tsv").
+		arg("--output_A2").arg("properties_a_secondary.tsv").
+		arg("--output_B1").arg("properties_b_primary.tsv").
+		arg("--output_B2").arg("properties_b_secondary.tsv").
+		saveFile("properties_a_primary.tsv").
+		saveFile("properties_a_secondary.tsv").
+		saveFile("properties_b_primary.tsv").
+		saveFile("properties_b_secondary.tsv")
+	scOutputCmd := scOutputProcessingBuilderCmd.run()
+
+	return {
+		// must have sampleId and scClonotypeKey columns
+		abundanceTsv: abundanceTsv,
+
+		// must have scClonotypeKey columns
+		propertiesAPrimaryTsv: scOutputCmd.getFile("properties_a_primary.tsv"),
+		propertiesASecondaryTsv: scOutputCmd.getFile("properties_a_secondary.tsv"),
+		propertiesBPrimaryTsv: scOutputCmd.getFile("properties_b_primary.tsv"),
+		propertiesBSecondaryTsv: scOutputCmd.getFile("properties_b_secondary.tsv")
+	}
+})