@platforma-open/milaboratories.mixcr-clonotyping-2.workflow 3.17.1 → 3.19.0

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/mixcr-clonotyping/mixcr-clonotyping/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.mixcr-clonotyping-2.workflow@3.17.1 build /home/runner/work/mixcr-clonotyping/mixcr-clonotyping/workflow
+> @platforma-open/milaboratories.mixcr-clonotyping-2.workflow@3.19.0 build /home/runner/work/mixcr-clonotyping/mixcr-clonotyping/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
   info: Skipping unknown file type: test/columns.test.ts

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @platforma-open/milaboratories.mixcr-clonotyping.workflow
 
+## 3.19.0
+
+### Minor Changes
+
+- 4f376a4: Columns with number of clonotypes dropped due stop codon or oof added to qc table. Dependencies updates
+
+## 3.18.0
+
+### Minor Changes
+
+- 070fc0b: Min quality checkbox, refactored error correction, dep updates
+
 ## 3.17.1
 
 ### Patch Changes

package/dist/tengo/lib/calculate-export-specs.lib.tengo

@@ -129,13 +129,18 @@ addSpec := func(columns, additionalSpec) {
 
 
 
-calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId) {
+calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, exportMinQuality) {
 	ops := exportSpecOpsFromPreset(presetSpecForBack)
 
 	assemblingFeature := ops.assemblingFeature
 	cellTags := ops.cellTags
 	hasUmi := ops.hasUmi
 	splitByC := ops.splitByC
+	
+
+	if is_undefined(exportMinQuality) {
+		exportMinQuality = false
+	}
 
 	isSingleCell := !is_undefined(cellTags) && len(cellTags) > 0
 	hashCellKey := isSingleCell && len(cellTags) > 1
@@ -482,7 +487,7 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId) {
 						}
 					} ]
 				exportArgs += [ [ "-" + alphabetShortMixcr + "Feature" + imputedU, featureInFrameU ] ]
-				if !isImputed && !isAminoAcid {
+				if !isImputed && !isAminoAcid && exportMinQuality {
 					columnsSpecPerClonotypeNoAggregates += [ {
 							column: "minQual" + featureU,
 							id: "min-qual-" + featureL,

package/dist/tengo/lib/qc-report-columns.lib.tengo

@@ -567,6 +567,38 @@ getQcReportColumns := func(hasUmi, isSingleCell, sampleIdAxisSpec, chains, cellT
             }
         }
     },
+    {
+        column: "assemble.clonotypesDroppedByStopCodons",
+        id: "assemble-clonotypes-dropped-by-stop-codons",
+        allowNA: true,
+        naRegex: "NaN",
+        spec: {
+            name: "mixcr.com/reports/assemble/clonotypesDroppedByStopCodons",
+            valueType: "Long",
+            annotations: {
+                "pl7.app/min": "0",
+                "pl7.app/table/orderPriority": "108200",
+                "pl7.app/table/visibility": "optional",
+                "pl7.app/label": "Clonotypes Dropped - Stop Codons"
+            }
+        }
+    },
+    {
+        column: "assemble.clonotypesDroppedByOutOfFrame",
+        id: "assemble-clonotypes-dropped-by-out-of-frame",
+        allowNA: true,
+        naRegex: "NaN",
+        spec: {
+            name: "mixcr.com/reports/assemble/clonotypesDroppedByOutOfFrame",
+            valueType: "Long",
+            annotations: {
+                "pl7.app/min": "0",
+                "pl7.app/table/orderPriority": "108100",
+                "pl7.app/table/visibility": "optional",
+                "pl7.app/label": "Clonotypes Dropped - Out of Frame"
+            }
+        }
+    },
     {
         column: "totalClonotypes",
         id: "total-clonotypes",
@@ -1248,6 +1280,44 @@ getQcReportColumns := func(hasUmi, isSingleCell, sampleIdAxisSpec, chains, cellT
         }
     }]
 
+
+    scDropOrder := 107700
+    for chain in chains {
+        singleCellColumns += [{
+            column: "assemble.clonotypesDroppedByOutOfFrameByChain." + chain,
+            id: "assemble-clonotypes-dropped-by-out-of-frame-by-chain-" + chain,
+            allowNA: true,
+            naRegex: "NaN",
+            spec: {
+                name: "mixcr.com/reports/assemble/clonotypesDroppedByOutOfFrameByChain/" + chain,
+                valueType: "Long",
+                annotations: {
+                    "pl7.app/min": "0",
+                    "pl7.app/table/orderPriority": string(scDropOrder),
+                    "pl7.app/table/visibility": "optional",
+                    "pl7.app/label": "Clonotypes Dropped - Out of Frame (" + chain + ")"
+                }
+            }
+        },
+        {
+            column: "assemble.clonotypesDroppedByStopCodonsByChain." + chain,
+            id: "assemble-clonotypes-dropped-by-stop-codons-by-chain-" + chain,
+            allowNA: true,
+            naRegex: "NaN",
+            spec: {
+                name: "mixcr.com/reports/assemble/clonotypesDroppedByStopCodonsByChain/" + chain,
+                valueType: "Long",
+                annotations: {
+                    "pl7.app/min": "0",
+                    "pl7.app/table/orderPriority": string(scDropOrder - 50),
+                    "pl7.app/table/visibility": "optional",
+                    "pl7.app/label": "Clonotypes Dropped - Stop Codons (" + chain + ")"
+                }
+            }
+        }]
+        scDropOrder -= 100
+    }
+
     for cellTag in cellTags {
         singleCellColumns += [{
             column: "refineTags." + cellTag + ".outputCount",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-open/milaboratories.mixcr-clonotyping-2.workflow",
-  "version": "3.17.1",
+  "version": "3.19.0",
   "description": "Tengo-based template",
   "dependencies": {
     "@platforma-sdk/workflow-tengo": "^5.6.6",

package/src/calculate-export-specs.lib.tengo

@@ -129,13 +129,18 @@ addSpec := func(columns, additionalSpec) {
 // Ordering rules
 //   AA Sequences
 
-calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId) {
+calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, exportMinQuality) {
 	ops := exportSpecOpsFromPreset(presetSpecForBack)
 
 	assemblingFeature := ops.assemblingFeature
 	cellTags := ops.cellTags
 	hasUmi := ops.hasUmi
 	splitByC := ops.splitByC
+	
+	// Default to false if not provided
+	if is_undefined(exportMinQuality) {
+		exportMinQuality = false
+	}
 
 	isSingleCell := !is_undefined(cellTags) && len(cellTags) > 0
 	hashCellKey := isSingleCell && len(cellTags) > 1
@@ -482,7 +487,7 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId) {
 						}
 					} ]
 				exportArgs += [ [ "-" + alphabetShortMixcr + "Feature" + imputedU, featureInFrameU ] ]
-				if !isImputed && !isAminoAcid {
+				if !isImputed && !isAminoAcid && exportMinQuality {
 					columnsSpecPerClonotypeNoAggregates += [ {
 							column: "minQual" + featureU,
 							id: "min-qual-" + featureL,

package/src/export-report.tpl.tengo

@@ -21,7 +21,6 @@ self.defineOutputs("qcReportTable")
 mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:main")
 ptablerSw := assets.importSoftware("@platforma-open/milaboratories.software-ptabler:main")
 
-
 self.body(func(inputs) {
     clnsData := inputs.clnsData
     presetSpecForBack := inputs.presetSpecForBack
@@ -36,6 +35,10 @@ self.body(func(inputs) {
     cellTags := presetSpecForBack.cellTags
     singleCellChainTsvsData := inputs.singleCellChainTsvsData
 
+    featureForFlags := "CDR3"
+    isOOFColumn := "isOOF" + featureForFlags
+    hasStopsColumn := "hasStopsIn" + featureForFlags
+
     chainInfos := {
 	"IGHeavy": { mixcrFilter: "IGH", name: "IG Heavy", shortName: "Heavy" },
 	"IGLight": { mixcrFilter: "IGK,IGL", name: "IG Light", shortName: "Light" },
@@ -127,6 +130,115 @@ self.body(func(inputs) {
     // Join counts and overwrite totalClonotypes to reflect exported (productive) clones
     joinedDf := processedDf.join(aggregatedCounts, { how: "left", on: ["sampleId"] })
 
+    // Count clonotypes filtered by stop codons and out-of-frame per sample
+    filterCountDfs := []
+    mixcrChainsArg := text.join(chainsForMixcr, ",")
+    for key, clnsFile in clnsFiles {
+        sampleId := json.decode(key)[0]
+        exportFiltersCmd := exec.builder().
+            inMediumQueue().
+            mem("16GiB").
+            cpu(2).
+            software(mixcrSw).
+            env("MI_USE_SYSTEM_CA", "true").
+            secret("MI_LICENSE", "MI_LICENSE").
+            arg("exportClones").
+            arg("--dont-split-files").
+            arg("--drop-default-fields").
+            arg("--reset-export-clone-table-splitting")
+        if mixcrChainsArg != "" {
+            exportFiltersCmd.arg("--chains").arg(mixcrChainsArg)
+        }
+        exportFiltersResult := exportFiltersCmd.
+            arg("-isOOF").arg(featureForFlags).
+            arg("-hasStops").arg(featureForFlags).
+            arg("clones.clns").
+            addFile("clones.clns", clnsFile).
+            arg("clones.tsv").
+            saveFile("clones.tsv").
+            cacheHours(3).
+            run()
+        filterTsv := exportFiltersResult.getFile("clones.tsv")
+        dfFilters := wf.frame(filterTsv, { xsvType: "tsv", inferSchema: false, schema: [ { column: isOOFColumn, type: "String" }, { column: hasStopsColumn, type: "String" } ] })
+        dfFilters = dfFilters.withColumns(
+            pt.when(pt.col(isOOFColumn).strToUpper().eq("TRUE")).then(pt.lit(1)).otherwise(pt.lit(0)).alias("__oof"),
+            pt.when(pt.col(hasStopsColumn).strToUpper().eq("TRUE")).then(pt.lit(1)).otherwise(pt.lit(0)).alias("__stop")
+        )
+        dfFilterCount := dfFilters.select(
+            pt.lit(sampleId).alias("sampleId"),
+            pt.col("__oof").sum().alias("assemble.clonotypesDroppedByOutOfFrame"),
+            pt.col("__stop").sum().alias("assemble.clonotypesDroppedByStopCodons")
+        )
+        filterCountDfs = append(filterCountDfs, dfFilterCount)
+    }
+
+    if len(filterCountDfs) > 0 {
+        filterCountsDf := len(filterCountDfs) > 1 ? pt.concat(filterCountDfs) : filterCountDfs[0]
+        joinedDf = joinedDf.join(filterCountsDf, { how: "left", on: ["sampleId"] })
+    } else {
+        joinedDf = joinedDf.withColumns(
+            pt.lit(0).alias("assemble.clonotypesDroppedByOutOfFrame"),
+            pt.lit(0).alias("assemble.clonotypesDroppedByStopCodons")
+        )
+    }
+
+    // Single-cell: per-chain counts of clonotypes dropped by stop codons / out-of-frame
+    if isSingleCell {
+        for chain in chains {
+            chainMixcr := chainInfos[chain].mixcrFilter
+            perChainFilterDfs := []
+            for key, clnsFile in clnsFiles {
+                sampleId := json.decode(key)[0]
+                exportChainFiltersCmd := exec.builder().
+                    inMediumQueue().
+                    mem("16GiB").
+                    cpu(2).
+                    software(mixcrSw).
+                    env("MI_USE_SYSTEM_CA", "true").
+                    secret("MI_LICENSE", "MI_LICENSE").
+                    arg("exportClones").
+                    arg("--dont-split-files").
+                    arg("--drop-default-fields").
+                    arg("--reset-export-clone-table-splitting").
+                    arg("--chains").arg(chainMixcr)
+                exportChainFiltersResult := exportChainFiltersCmd.
+                    arg("-isOOF").arg(featureForFlags).
+                    arg("-hasStops").arg(featureForFlags).
+                    arg("clones.clns").
+                    addFile("clones.clns", clnsFile).
+                    arg("clones.tsv").
+                    saveFile("clones.tsv").
+                    cacheHours(3).
+                    run()
+                chainFilterTsv := exportChainFiltersResult.getFile("clones.tsv")
+                dfChainFilters := wf.frame(chainFilterTsv, { xsvType: "tsv", inferSchema: false, schema: [ { column: isOOFColumn, type: "String" }, { column: hasStopsColumn, type: "String" } ] })
+                dfChainFilters = dfChainFilters.withColumns(
+                    pt.when(pt.col(isOOFColumn).strToUpper().eq("TRUE")).then(pt.lit(1)).otherwise(pt.lit(0)).alias("__oof"),
+                    pt.when(pt.col(hasStopsColumn).strToUpper().eq("TRUE")).then(pt.lit(1)).otherwise(pt.lit(0)).alias("__stop")
+                )
+                dfChainCount := dfChainFilters.select(
+                    pt.lit(sampleId).alias("sampleId"),
+                    pt.col("__oof").sum().alias("assemble.clonotypesDroppedByOutOfFrameByChain." + chain),
+                    pt.col("__stop").sum().alias("assemble.clonotypesDroppedByStopCodonsByChain." + chain)
+                )
+                perChainFilterDfs = append(perChainFilterDfs, dfChainCount)
+            }
+            if len(perChainFilterDfs) > 0 {
+                chainDf := len(perChainFilterDfs) > 1 ? pt.concat(perChainFilterDfs) : perChainFilterDfs[0]
+                chainAgg := chainDf.groupBy("sampleId").agg(
+                    pt.col("assemble.clonotypesDroppedByOutOfFrameByChain." + chain).sum().alias("assemble.clonotypesDroppedByOutOfFrameByChain." + chain),
+                    pt.col("assemble.clonotypesDroppedByStopCodonsByChain." + chain).sum().alias("assemble.clonotypesDroppedByStopCodonsByChain." + chain)
+                )
+                joinedDf = joinedDf.join(chainAgg, { how: "left", on: ["sampleId"] })
+            } else {
+                joinedDf = joinedDf.withColumns(
+                    pt.lit(0).alias("assemble.clonotypesDroppedByOutOfFrameByChain." + chain),
+                    pt.lit(0).alias("assemble.clonotypesDroppedByStopCodonsByChain." + chain)
+                )
+            }
+        }
+    }
+
     // Per-chain clonotype counts
     perChainJoined := joinedDf
     for chain in chains {
@@ -254,6 +366,19 @@ self.body(func(inputs) {
         }
     }
 
+    finalDf = finalDf.withColumns(
+        pt.col("assemble.clonotypesDroppedByOutOfFrame").fillNull(0).cast("Long").alias("assemble.clonotypesDroppedByOutOfFrame"),
+        pt.col("assemble.clonotypesDroppedByStopCodons").fillNull(0).cast("Long").alias("assemble.clonotypesDroppedByStopCodons")
+    )
+    if isSingleCell {
+        for chain in chains {
+            finalDf = finalDf.withColumns(
+                pt.col("assemble.clonotypesDroppedByOutOfFrameByChain." + chain).fillNull(0).cast("Long").alias("assemble.clonotypesDroppedByOutOfFrameByChain." + chain),
+                pt.col("assemble.clonotypesDroppedByStopCodonsByChain." + chain).fillNull(0).cast("Long").alias("assemble.clonotypesDroppedByStopCodonsByChain." + chain)
+            )
+        }
+    }
+
     for chain in chains {
         col := "clonotypesByChain." + chain
         finalDf = finalDf.withColumns(

package/src/main.tpl.tengo

@@ -122,7 +122,7 @@ wf.body(func(args) {
 			limitInput: limitInput,
 			perProcessMemGB: perProcessMemGB,
 			perProcessCPUs: perProcessCPUs,
-				highDiversityLibrary: args.highDiversityLibrary,
+				cloneClusteringMode: args.cloneClusteringMode,
 			blockId: blockId,
 			libraryId: libraryId,
 			presetCommonName: args.presetCommonName,
@@ -131,7 +131,8 @@ wf.body(func(args) {
 				rightAlignmentMode: args.rightAlignmentMode,
 				materialType: args.materialType,
 				tagPattern: args.tagPattern,
-				assembleClonesBy: args.assembleClonesBy
+				assembleClonesBy: args.assembleClonesBy,
+				exportMinQuality: args.exportMinQuality
 		})
 	})
 

package/src/mixcr-analyze.tpl.tengo

@@ -143,14 +143,19 @@ self.body(func(inputs) {
 	if !is_undefined(assembleClonesBy) {
 		mixcrCmdBuilder.arg("--assemble-clonotypes-by").arg(assembleClonesBy)
 	}
-	// High diversity library tuning
-	highDiversityLibrary := params.highDiversityLibrary
-	if !is_undefined(highDiversityLibrary) && highDiversityLibrary {
+	// Clone clustering tuning
+	cloneClusteringMode := params.cloneClusteringMode
+	if is_undefined(cloneClusteringMode) {
+		cloneClusteringMode = "default"
+	}
+	if cloneClusteringMode == "relaxed" {
 		mixcrCmdBuilder.
 			arg("-Massemble.cloneAssemblerParameters.minimalQuality=20").
 			arg("-Massemble.cloneAssemblerParameters.badQualityThreshold=10").
 			arg("-Massemble.cloneAssemblerParameters.cloneClusteringParameters.searchParameters=oneMismatchOrIndel").
 			arg("-Massemble.cloneAssemblerParameters.cloneClusteringParameters.searchDepth=1")
+	} else if cloneClusteringMode == "off" {
+		mixcrCmdBuilder.arg("-Massemble.cloneAssemblerParameters.cloneClusteringParameters=null")
 	}
 
 	inputMap := inputData.inputs()

package/src/process.tpl.tengo

@@ -203,7 +203,7 @@ self.body(func(inputs) {
 		}
 	} ]
 
-	exportSpecs := calculateExportSpecs(presetSpecForBack, sampleIdAxisSpec, blockId)
+	exportSpecs := calculateExportSpecs(presetSpecForBack, sampleIdAxisSpec, blockId, params.exportMinQuality)
 
 	columnsSpecPerSample := exportSpecs.columnsSpecPerSample
 	columnsSpecPerSampleSc := exportSpecs.columnsSpecPerSampleSc
@@ -273,7 +273,7 @@ self.body(func(inputs) {
 					materialType: params.materialType,
 					tagPattern: params.tagPattern,
 					assembleClonesBy: params.assembleClonesBy,
-					highDiversityLibrary: params.highDiversityLibrary
+					cloneClusteringMode: params.cloneClusteringMode
 				},
 				library: library,
 				presetContent: presetContent

package/src/qc-report-columns.lib.tengo

@@ -567,6 +567,38 @@ getQcReportColumns := func(hasUmi, isSingleCell, sampleIdAxisSpec, chains, cellT
             }
         }
     },
+    {
+        column: "assemble.clonotypesDroppedByStopCodons",
+        id: "assemble-clonotypes-dropped-by-stop-codons",
+        allowNA: true,
+        naRegex: "NaN",
+        spec: {
+            name: "mixcr.com/reports/assemble/clonotypesDroppedByStopCodons",
+            valueType: "Long",
+            annotations: {
+                "pl7.app/min": "0",
+                "pl7.app/table/orderPriority": "108200",
+                "pl7.app/table/visibility": "optional",
+                "pl7.app/label": "Clonotypes Dropped - Stop Codons"
+            }
+        }
+    },
+    {
+        column: "assemble.clonotypesDroppedByOutOfFrame",
+        id: "assemble-clonotypes-dropped-by-out-of-frame",
+        allowNA: true,
+        naRegex: "NaN",
+        spec: {
+            name: "mixcr.com/reports/assemble/clonotypesDroppedByOutOfFrame",
+            valueType: "Long",
+            annotations: {
+                "pl7.app/min": "0",
+                "pl7.app/table/orderPriority": "108100",
+                "pl7.app/table/visibility": "optional",
+                "pl7.app/label": "Clonotypes Dropped - Out of Frame"
+            }
+        }
+    },
     {
         column: "totalClonotypes",
         id: "total-clonotypes",
@@ -1248,6 +1280,44 @@ getQcReportColumns := func(hasUmi, isSingleCell, sampleIdAxisSpec, chains, cellT
         }
     }]
 
+    // Single-cell per-chain dropped clonotypes by OOF / stop codons
+    scDropOrder := 107700
+    for chain in chains {
+        singleCellColumns += [{
+            column: "assemble.clonotypesDroppedByOutOfFrameByChain." + chain,
+            id: "assemble-clonotypes-dropped-by-out-of-frame-by-chain-" + chain,
+            allowNA: true,
+            naRegex: "NaN",
+            spec: {
+                name: "mixcr.com/reports/assemble/clonotypesDroppedByOutOfFrameByChain/" + chain,
+                valueType: "Long",
+                annotations: {
+                    "pl7.app/min": "0",
+                    "pl7.app/table/orderPriority": string(scDropOrder),
+                    "pl7.app/table/visibility": "optional",
+                    "pl7.app/label": "Clonotypes Dropped - Out of Frame (" + chain + ")"
+                }
+            }
+        },
+        {
+            column: "assemble.clonotypesDroppedByStopCodonsByChain." + chain,
+            id: "assemble-clonotypes-dropped-by-stop-codons-by-chain-" + chain,
+            allowNA: true,
+            naRegex: "NaN",
+            spec: {
+                name: "mixcr.com/reports/assemble/clonotypesDroppedByStopCodonsByChain/" + chain,
+                valueType: "Long",
+                annotations: {
+                    "pl7.app/min": "0",
+                    "pl7.app/table/orderPriority": string(scDropOrder - 50),
+                    "pl7.app/table/visibility": "optional",
+                    "pl7.app/label": "Clonotypes Dropped - Stop Codons (" + chain + ")"
+                }
+            }
+        }]
+        scDropOrder -= 100
+    }
+
     for cellTag in cellTags {
         singleCellColumns += [{
             column: "refineTags." + cellTag + ".outputCount",

package/src/test/columns-calculate.tpl.tengo

@@ -9,7 +9,7 @@ self.defineOutputs("exportSpecs")
 self.body(func(inputs) {
 	presetSpecForBack := inputs.presetSpecForBack.getDataAsJson()
 
-	exportSpecs := calculateExportSpecs(presetSpecForBack, {}, "test")
+	exportSpecs := calculateExportSpecs(presetSpecForBack, {}, "test", false)
 	exportSpecs = maps.deepMerge(exportSpecs, {
     	axisByClonotypeKeyGen: undefined,
 		axisByScClonotypeKeyGen: undefined,

package/src/test/columns.test.tpl.tengo

@@ -7,7 +7,7 @@ render := import("@platforma-sdk/workflow-tengo:render")
 calculatePresetInfoTpl := assets.importTemplate(":calculate-preset-info")
 columnsCalculateTpl := assets.importTemplate(":test.columns-calculate")
 
-self.defineOutputs("conf")
+self.defineOutputs("exportSpecs")
 
 self.body(func(inputs) {
 	preset := inputs.preset