@platforma-open/milaboratories.mixcr-clonotyping-2.workflow 2.18.2 → 2.19.0

package/src/process-single-cell.tpl.tengo

@@ -4,14 +4,14 @@ pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 exec := import("@platforma-sdk/workflow-tengo:exec")
 maps := import("@platforma-sdk/workflow-tengo:maps")
-
+slices := import("@platforma-sdk/workflow-tengo:slices")
+clonotypeLabel := import(":clonotype-label")
 json := import("json")
+pt := import("@platforma-sdk/workflow-tengo:pt")
 
 self.defineOutputs("abundanceTsv", "clonotypeTsv", "propertiesAPrimaryTsv", "propertiesASecondaryTsv", "propertiesBPrimaryTsv", "propertiesBSecondaryTsv")
 
-scGroupBuilderSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-clonotyping-2.single-cell-scripts:sc-group-builder")
-scPreprocessingSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-clonotyping-2.single-cell-scripts:preprocessing")
-scOutputProcessingSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-clonotyping-2.single-cell-scripts:output-processing")
+ptablerSw := assets.importSoftware("@platforma-open/milaboratories.software-ptabler:main")
 
 self.body(func(inputs) {
 	byCellTagA := inputs[pConstants.VALUE_FIELD_NAME]
@@ -22,93 +22,277 @@ self.body(func(inputs) {
 	propertiesA := inputs.propertiesA
 	propertiesB := inputs.propertiesB
 
-	// Creating files map
-	filesMap := {}
-	inputMapA := byCellTagA.inputs()
-	for k in maps.getKeys(inputMapA) {
-		v := inputMapA[k]
-		key := json.decode(k)
-		fileName := "by_cell_a_" + key[0] + ".tsv"
-		filesMap[fileName] = v
-	}
+	mainAbundanceColumn := inputs.params.mainAbundanceColumn
+	mainIsProductiveColumn := inputs.params.mainIsProductiveColumn
+
+	schemaPerClonotypeNoAggregates := inputs.params.schemaPerClonotypeNoAggregates
+
+	//
+	// Preprocessing
+	//
+
+	/**
+	 * Universal preprocessing step for A and B chain files
+	 * @param byCellTag: Map<string:[sampleId], tsv_by_cell_tag>
+	 * @return the output file
+	 */
+	preprocessByCell := func(byCellTag) {
+		wf := pt.workflow()
+		sampleDataFrames := []
+
+		inputMap := byCellTag.inputs()
+		maps.forEach(inputMap, func(sKey, inputFile) {
+			key := json.decode(sKey)
+			ll.assert(len(key) == 1, "preprocessByCell: byCellTag key should have one element, got %v", key)
+			sampleId := key[0]
+
+			// Create a DataFrame for the current sample's file
+			sampleDf := wf.frame(inputFile, {
+				xsvType: "tsv",
+				schema: [
+					{ column: "cellKey", type: "String" },
+					{ column: "clonotypeKey", type: "String" },
+					{ column: mainAbundanceColumn, type: "Long" }, // Ensure mainAbundanceColumn is treated as Long
+					{ column: mainIsProductiveColumn, type: "String" } // Keep as String for direct comparison
+				],
+				inferSchema: false // Disable further inference so columns not defined in schema are interpreted as strings
+			}).withColumns(
+				pt.col("cellKey"), pt.col("clonotypeKey"), pt.col(mainAbundanceColumn), pt.col(mainIsProductiveColumn)
+			)
+
+			// Add sampleId column
+			sampleDfWithId := sampleDf.addColumns(
+				pt.lit(sampleId).alias("sampleId")
+			)
+			sampleDataFrames = append(sampleDataFrames, sampleDfWithId)
+		})
+
+		ll.assert(len(sampleDataFrames) > 0, "No input files to process in preprocessByCell")
+
+		concatenatedDf := pt.concat(sampleDataFrames)
+
+		// Add rawChainRank column
+		dfWithRawRank := concatenatedDf.withColumns(
+			pt.rank(pt.col(mainAbundanceColumn), {descending: true}).
+				over([pt.col("sampleId"), pt.col("cellKey")]).
+				alias("rawChainRank")
+		)
+
+		// Filter by rawChainRank <= 2
+		dfFilteredByRawRank := dfWithRawRank.filter(
+			pt.col("rawChainRank").le(2)
+		)
+
+		// Prepare expressions for chainRank ordering
+		// 1. isProductiveNumeric: "True" -> 0, "False" -> 1 (for ascending sort)
+		isProductiveNumericExpr := pt.when(pt.col(mainIsProductiveColumn).eq("True")).
+			then(pt.lit(0)).
+			otherwise(pt.lit(1))
 
-	inputMapB := byCellTagB.inputs()
-	for k in maps.getKeys(inputMapB) {
-		v := inputMapB[k]
-		key := json.decode(k)
-		fileName := "by_cell_b_" + key[0] + ".tsv"
-		filesMap[fileName] = v
+		// 2. negativeAbundance: mainAbundanceColumn * -1 (for ascending sort by this, meaning descending by original abundance)
+		negativeAbundanceExpr := pt.col(mainAbundanceColumn).multiply(-1)
+
+		// Add chainRank column and remove temp column rawChainRank
+		dfWithChainRank := dfFilteredByRawRank.withColumns(
+			pt.rank([
+					isProductiveNumericExpr,
+					negativeAbundanceExpr
+				], {descending: false}).
+				over([pt.col("sampleId"), pt.col("cellKey")]).
+				alias("chainRank")
+		).withoutColumns("rawChainRank")
+
+		dfWithChainRank.save("output.tsv")
+
+		// Run the workflow
+		runResult := wf.run()
+		return runResult.getFile("output.tsv")
 	}
 
-	scPreprocessingCmd := exec.builder().
-		printErrStreamToStdout().
-		software(scPreprocessingSw)
+	chainAOutput := preprocessByCell(byCellTagA)
+	chainBOutput := preprocessByCell(byCellTagB)
+
+	//
+	// Cell grouping - Reimplemented with PTabler pt API
+	//
+	cellGroupingWf := pt.workflow()
 
-	for name in maps.getKeys(filesMap) {
-		scPreprocessingCmd.addFile(name, filesMap[name]).arg(name)
+	chainATableDf := cellGroupingWf.frame(chainAOutput, {xsvType: "tsv"})
+	chainBTableDf := cellGroupingWf.frame(chainBOutput, {xsvType: "tsv"})
+
+	// Dynamically generate filter steps for chains A and B, ranks 1 and 2
+	// Store resulting DataFrames in a map for easier access
+	rankedDfs := {}
+	for chainData in [{prefix: "a", df: chainATableDf}, {prefix: "b", df: chainBTableDf}] {
+		for rankVal in [1, 2] {
+			dfKey := "chain_" + chainData.prefix + "_rank" + string(rankVal) + "_df"
+			rankedDfs[dfKey] = chainData.df.filter(pt.col("chainRank").eq(rankVal))
+		}
 	}
 
-	// Data preprocessing
-	scPreprocessingCmd = scPreprocessingCmd.arg("--output_chainA").arg("chain_a_output.tsv").
-		arg("--output_chainB").arg("chain_b_output.tsv").
-		saveFile("chain_a_output.tsv").
-		saveFile("chain_b_output.tsv")
-	run := scPreprocessingCmd.run()
-	chainAoutput := run.getFile("chain_a_output.tsv")
-	chainBoutput := run.getFile("chain_b_output.tsv")
-
-
-	// Generating scClonotypeKey and abundanceTable
-	scClonotypeBuilderCmd := exec.builder().
-		printErrStreamToStdout().
-		software(scGroupBuilderSw).
-		addFile("chain_a_output.tsv", chainAoutput).
-		addFile("chain_b_output.tsv", chainBoutput).
-		arg("--only_full_clonotypes").
-		arg("--chainA").arg("chain_a_output.tsv").
-		arg("--chainB").arg("chain_b_output.tsv").
-		arg("--output_clonotype").arg("clonotype.tsv").
-		arg("--output_cell").arg("abundance.tsv").
-		saveFile("clonotype.tsv").
-		saveFile("abundance.tsv")
-	scClonotypeCmd := scClonotypeBuilderCmd.run()
-
-	clonotypeTsv := scClonotypeCmd.getFile("clonotype.tsv")
-	abundanceTsv := scClonotypeCmd.getFile("abundance.tsv")
+	chainAMergedDf := rankedDfs["chain_a_rank1_df"].join(rankedDfs["chain_a_rank2_df"], {
+		how: "full",
+		on: ["sampleId", "cellKey"],
+		coalesce: true,
+		leftColumns: [{ column: "clonotypeKey", rename: "clonotypeKeyA1" }],
+		rightColumns: [{ column: "clonotypeKey", rename: "clonotypeKeyA2" }]
+	})
+
+	chainBMergedDf := rankedDfs["chain_b_rank1_df"].join(rankedDfs["chain_b_rank2_df"], {
+		how: "full",
+		on: ["sampleId", "cellKey"],
+		coalesce: true,
+		leftColumns: [{ column: "clonotypeKey", rename: "clonotypeKeyB1" }],
+		rightColumns: [{ column: "clonotypeKey", rename: "clonotypeKeyB2" }]
+	})
+
+	allChainsMergedDf := chainAMergedDf.join(chainBMergedDf, {
+		how: "full",
+		on: ["sampleId", "cellKey"],
+		coalesce: true
+	})
+
+	scClonotypeKeyExpr := pt.concatStr(
+		[
+			pt.col("clonotypeKeyA1").fillNull("NA"),
+			pt.col("clonotypeKeyA2").fillNull("NA"),
+			pt.col("clonotypeKeyB1").fillNull("NA"),
+			pt.col("clonotypeKeyB2").fillNull("NA")
+		],
+		{delimiter: "#"}
+	).hash("sha256", "base64_alphanumeric", 120).alias("scClonotypeKey")
+
+	allChainsMergedWithScKeyDf := allChainsMergedDf.withColumns(scClonotypeKeyExpr)
+
+	filterCondition := pt.and(
+		pt.col("clonotypeKeyA1").isNotNull(),
+		pt.col("clonotypeKeyB1").isNotNull()
+	)
+	allChainsFilteredDf := allChainsMergedWithScKeyDf.filter(filterCondition)
+
+	clonotypeTableDf := allChainsFilteredDf.groupBy(
+		"scClonotypeKey", "clonotypeKeyA1", "clonotypeKeyA2", "clonotypeKeyB1", "clonotypeKeyB2"
+	).agg(
+		pt.col("sampleId").nUnique().alias("sampleCount")
+	)
+
+	clonotypeTableDf = clonotypeLabel.addClonotypeLabelColumnsPt(clonotypeTableDf, "scClonotypeKey", "clonotypeLabel", pt)
+
+	clonotypeTableDf.save("clonotype.tsv")
+
+	cellCountsDf := allChainsFilteredDf.groupBy("sampleId", "scClonotypeKey").agg(
+		pt.col("cellKey").count().alias("uniqueCellCount")
+	)
+
+	cellCountsWithFractionDf := cellCountsDf.withColumns(
+		pt.col("uniqueCellCount").truediv(
+			pt.col("uniqueCellCount").sum().over("sampleId")
+		).alias("uniqueCellFraction")
+	)
+
+	cellCountsWithFractionDf.save("abundance.tsv")
+
+	cellGroupingRunResult := cellGroupingWf.run()
+
+	clonotypeTsv := cellGroupingRunResult.getFile("clonotype.tsv")
+	abundanceTsv := cellGroupingRunResult.getFile("abundance.tsv")
+
+	//
+	// Output processing - Reimplemented with PTabler pt API
+	//
 
 	propertiesAFile := propertiesA.inputs()["[]"]
 	propertiesBFile := propertiesB.inputs()["[]"]
 
-	// Propagate scClonotypeKey to properties tables
-	scOutputProcessingBuilderCmd := exec.builder().
-		printErrStreamToStdout().
-		software(scOutputProcessingSw).
-		addFile("clonotype.tsv", clonotypeTsv).
-		arg("--main_table").arg("clonotype.tsv").
-		addFile("properties_a.tsv", propertiesAFile).
-		addFile("properties_b.tsv", propertiesBFile).
-		arg("--properties_a").arg("properties_a.tsv").
-		arg("--properties_b").arg("properties_b.tsv").
-		arg("--output_A1").arg("properties_a_primary.tsv").
-		arg("--output_A2").arg("properties_a_secondary.tsv").
-		arg("--output_B1").arg("properties_b_primary.tsv").
-		arg("--output_B2").arg("properties_b_secondary.tsv").
-		saveFile("properties_a_primary.tsv").
-		saveFile("properties_a_secondary.tsv").
-		saveFile("properties_b_primary.tsv").
-		saveFile("properties_b_secondary.tsv")
-	scOutputCmd := scOutputProcessingBuilderCmd.run()
+	outputProcessingWf := pt.workflow()
+
+	// Schema for clonotype.tsv (main_clonotypes table)
+	clonotypeTableSchema := [
+		{ column: "scClonotypeKey", type: "String" },
+		{ column: "clonotypeKeyA1", type: "String" },
+		{ column: "clonotypeKeyA2", type: "String" },
+		{ column: "clonotypeKeyB1", type: "String" },
+		{ column: "clonotypeKeyB2", type: "String" },
+		{ column: "sampleCount", type: "Int" }
+	]
+
+	mainClonotypesDf := outputProcessingWf.frame(clonotypeTsv, {
+		xsvType: "tsv",
+		schema: clonotypeTableSchema,
+		inferSchema: false
+	})
+
+	propsASchema := [{ column: "clonotypeKey", type: "String" }]
+	propsADf := outputProcessingWf.frame(propertiesAFile, {
+		xsvType: "tsv",
+		schema: propsASchema,
+		inferSchema: false
+	})
+
+	propsBSchema := [{ column: "clonotypeKey", type: "String" }]
+	propsBDf := outputProcessingWf.frame(propertiesBFile, {
+		xsvType: "tsv",
+		id: "props_b",
+		schema: propsBSchema,
+		inferSchema: false
+	})
+
+
+	clonotypeColumnNames := []
+	for cc in schemaPerClonotypeNoAggregates {
+		clonotypeColumnNames = append(clonotypeColumnNames, cc.column)
+	}
+
+	finalOutputColumns := ["scClonotypeKey", "clonotypeKey"] + clonotypeColumnNames
+
+
+	chainMappings := [
+		{ chainKeyCol: "clonotypeKeyA1", propsTable: "props_a", internalOutTable: "props_a1_joined", finalOutFile: "properties_a_primary.tsv" },
+		{ chainKeyCol: "clonotypeKeyA2", propsTable: "props_a", internalOutTable: "props_a2_joined", finalOutFile: "properties_a_secondary.tsv" },
+		{ chainKeyCol: "clonotypeKeyB1", propsTable: "props_b", internalOutTable: "props_b1_joined", finalOutFile: "properties_b_primary.tsv" },
+		{ chainKeyCol: "clonotypeKeyB2", propsTable: "props_b", internalOutTable: "props_b2_joined", finalOutFile: "properties_b_secondary.tsv" }
+	]
+
+	propsMapDfs := {
+		"props_a": propsADf,
+		"props_b": propsBDf
+	}
+
+	for mapping in chainMappings {
+		filteredClonotypesDf := mainClonotypesDf.filter(
+			pt.col(mapping.chainKeyCol).isNotNull()
+		)
+
+		filteredClonotypesWithKeyDf := filteredClonotypesDf.withColumns(
+			pt.col(mapping.chainKeyCol).alias("clonotypeKey")
+		)
+
+		leftDfForJoin := propsMapDfs[mapping.propsTable]
+
+		joinedDf := leftDfForJoin.join(filteredClonotypesWithKeyDf, {
+			how: "inner",
+			on: ["clonotypeKey"]
+		})
+
+		joinedDf.save(mapping.finalOutFile, {
+			columns: finalOutputColumns,
+			xsvType: "tsv"
+		})
+	}
+
+	outputProcessingRunResult := outputProcessingWf.run()
 
 	return {
 		// must have sampleId and scClonotypeKey columns
 		abundanceTsv: abundanceTsv,
-		// used for aggregates (i.e. sampleCount)
+
+		// used for aggregates (i.e. sampleCount and clonotypeLabel)
 		clonotypeTsv: clonotypeTsv,
 
 		// must have scClonotypeKey columns
-		propertiesAPrimaryTsv: scOutputCmd.getFile("properties_a_primary.tsv"),
-		propertiesASecondaryTsv: scOutputCmd.getFile("properties_a_secondary.tsv"),
-		propertiesBPrimaryTsv: scOutputCmd.getFile("properties_b_primary.tsv"),
-		propertiesBSecondaryTsv: scOutputCmd.getFile("properties_b_secondary.tsv")
+		propertiesAPrimaryTsv: outputProcessingRunResult.getFile(chainMappings[0].finalOutFile),
+		propertiesASecondaryTsv: outputProcessingRunResult.getFile(chainMappings[1].finalOutFile),
+		propertiesBPrimaryTsv: outputProcessingRunResult.getFile(chainMappings[2].finalOutFile),
+		propertiesBSecondaryTsv: outputProcessingRunResult.getFile(chainMappings[3].finalOutFile)
 	}
 })

package/src/process.tpl.tengo

@@ -7,6 +7,7 @@ assets := import("@platforma-sdk/workflow-tengo:assets")
 pframes := import("@platforma-sdk/workflow-tengo:pframes")
 slices := import("@platforma-sdk/workflow-tengo:slices")
 maps := import("@platforma-sdk/workflow-tengo:maps")
+sets := import("@platforma-sdk/workflow-tengo:sets")
 
 calculateExportSpecs := import(":calculate-export-specs")
 
@@ -200,9 +201,9 @@ self.body(func(inputs) {
 
 	columnsSpecPerSample := exportSpecs.columnsSpecPerSample
 	columnsSpecPerSampleSc := exportSpecs.columnsSpecPerSampleSc
-	columnsSpecPerClonotype := exportSpecs.columnsSpecPerClonotype
+	columnsSpecPerClonotypeNoAggregates := exportSpecs.columnsSpecPerClonotypeNoAggregates
+	columnsSpecPerClonotypeAggregates := exportSpecs.columnsSpecPerClonotypeAggregates
 	columnsSpecPerClonotypeSc := exportSpecs.columnsSpecPerClonotypeSc
-	// columnsSpec := exportSpecs.columnsSpec
 
 	clonotypeKeyColumns := exportSpecs.clonotypeKeyColumns
 	clonotypeKeyArgs := exportSpecs.clonotypeKeyArgs
@@ -214,8 +215,13 @@ self.body(func(inputs) {
 
 	exportArgs := exportSpecs.exportArgs
 
+	mainIsProductiveColumn := exportSpecs.mainIsProductiveColumn
+	mainIsProductiveArgs := exportSpecs.mainIsProductiveArgs
+
 	mainAbundanceColumnNormalized := exportSpecs.mainAbundanceColumnNormalized
 	mainAbundanceColumnUnnormalized := exportSpecs.mainAbundanceColumnUnnormalized
+	mainAbundanceColumnNormalizedArgs := exportSpecs.mainAbundanceColumnNormalizedArgs
+	mainAbundanceColumnUnnormalizedArgs := exportSpecs.mainAbundanceColumnUnnormalizedArgs
 
 	if is_undefined(axesByClonotypeKey) {
 		ll.panic("Absent clonotype key not supported")
@@ -284,6 +290,19 @@ self.body(func(inputs) {
 
 	perChainResults := {}
 
+	// mapping removing duplicates
+	columnsToSchema := func(columns) {
+		schema := []
+		columnsAdded := {}
+		for col in columns {
+			if !columnsAdded[col.column] {
+				schema += [ { column: col.column, type: col.spec.valueType } ]
+				columnsAdded[col.column] = true
+			}
+		}
+		return schema
+	}
+
 	for chain in chains {
 		chainInfo := chainInfos[chain]
 		ll.assert(!is_undefined(chainInfo), "chainInfo not found for chain %v", chain)
@@ -356,6 +375,14 @@ self.body(func(inputs) {
 						chains: chainInfo.mixcrFilter,
 						clonotypeKeyColumns: clonotypeKeyColumns,
 						clonotypeKeyArgs: clonotypeKeyArgs,
+
+						mainIsProductiveColumn: mainIsProductiveColumn,
+						mainIsProductiveArgs: mainIsProductiveArgs,
+						mainAbundanceColumnNormalized: mainAbundanceColumnNormalized,
+						mainAbundanceColumnUnnormalized: mainAbundanceColumnUnnormalized,
+						mainAbundanceColumnNormalizedArgs: mainAbundanceColumnNormalizedArgs,
+						mainAbundanceColumnUnnormalizedArgs: mainAbundanceColumnUnnormalizedArgs,
+
 						cellTagColumns: cellTagColumns,
 						exportArgs: exportArgs,
 						isLibraryFileGzipped: isLibraryFileGzipped
@@ -392,7 +419,7 @@ self.body(func(inputs) {
 				xsvType: "tsv",
 				settings: {
 					axes: axesByClonotypeKeyWithChain,
-					columns: columnsSpecPerClonotype,
+					columns: columnsSpecPerClonotypeNoAggregates + columnsSpecPerClonotypeAggregates,
 					storageFormat: "Binary",
 					partitionKeyLength: 0
 				},
@@ -411,9 +438,9 @@ self.body(func(inputs) {
 					params: {
 						mainAbundanceColumnNormalized: mainAbundanceColumnNormalized,
 						mainAbundanceColumnUnnormalized: mainAbundanceColumnUnnormalized,
-						clonotypeColumns: slices.map(columnsSpecPerClonotype, func(col) {
-							return col.column
-						})
+						schemaPerClonotypeNoAggregates: columnsToSchema(columnsSpecPerClonotypeNoAggregates),
+						schemaPerClonotypeAggregates: columnsToSchema(columnsSpecPerClonotypeAggregates),
+						schemaPerSample: columnsToSchema(columnsSpecPerSample)
 					}
 				}
 			}
@@ -480,7 +507,7 @@ self.body(func(inputs) {
 
 			//removing columns from secondary chain except aaSeqCDR3
 			columnsSpecPerClonotypeSecondary := []
-			for col in columnsSpecPerClonotype {
+			for col in columnsSpecPerClonotypeNoAggregates {
 				if col.column == "aaSeqCDR3" {
 					columnsSpecPerClonotypeSecondary += [ col ]
 				}
@@ -530,7 +557,7 @@ self.body(func(inputs) {
 						xsvType: "tsv",
 						settings: {
 							axes: axesByScClonotypeKeyWithReceptor,
-							columns: transformSpecs(isPrimary ? columnsSpecPerClonotype : columnsSpecPerClonotypeSecondary, {
+							columns: transformSpecs(isPrimary ? columnsSpecPerClonotypeNoAggregates : columnsSpecPerClonotypeSecondary, {
 								spec: {
 									domain: {
 										"pl7.app/vdj/scClonotypeChain": chainLetterU,
@@ -563,7 +590,12 @@ self.body(func(inputs) {
 					extra: {
 						byCellTagB: perChainResults[chainB].tsvForSingleCell.data,
 						propertiesA: perChainResults[chainA].clonotypeProperties.data,
-						propertiesB: perChainResults[chainB].clonotypeProperties.data
+						propertiesB: perChainResults[chainB].clonotypeProperties.data,
+						params: {
+							mainAbundanceColumn: mainAbundanceColumnUnnormalized,
+							mainIsProductiveColumn: mainIsProductiveColumn,
+							schemaPerClonotypeNoAggregates: columnsToSchema(columnsSpecPerClonotypeNoAggregates)
+						}
 					}
 				}
 			)