@platforma-open/milaboratories.mixcr-clonotyping-2.workflow 3.26.3 → 3.27.1

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/mixcr-clonotyping/mixcr-clonotyping/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.mixcr-clonotyping-2.workflow@3.26.3 build /home/runner/work/mixcr-clonotyping/mixcr-clonotyping/workflow
+> @platforma-open/milaboratories.mixcr-clonotyping-2.workflow@3.27.1 build /home/runner/work/mixcr-clonotyping/mixcr-clonotyping/workflow
 > shx rm -rf dist && pl-tengo check && pl-tengo build
 
   info: Skipping unknown file type: test/columns.test.ts

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @platforma-open/milaboratories.mixcr-clonotyping.workflow
 
+## 3.27.1
+
+### Patch Changes
+
+- a297368: Size MiXCR analyze memory from the input reads' file size on top of the per-analysis floor. The per-analysis baseline (64 / 110 / 192 GiB for default / contig-cell / single-cell+MiTool) stays as a floor; memory then grows linearly with the R1+R2 FASTQ blob size (4 bytes of RAM per byte), clamped to 256 GiB. File size is read from blob metadata (getBlobSize) with no pre-exec. The explicit "Advanced Settings" memory override is unchanged; on backends that can't evaluate resource formulas the baseline is the static fallback.
+
+## 3.27.0
+
+### Minor Changes
+
+- 1ed23a7: Add "Impute non-covered parts from germline" option for generic amplicon presets. When enabled, the block exports additional germline-imputed sequence columns for the gene-feature regions outside the assembling feature span (plus the full VDJRegion), reconstructing non-covered parts from the assigned V/J germline. Imputed sequences are not used for clonotype assembly. The option is shown only for presets that expose "Assemble clones by".
+
 ## 3.26.3
 
 ### Patch Changes

package/dist/tengo/lib/calculate-export-specs.lib.tengo

@@ -105,6 +105,71 @@ formatId := func(input) {
     return result
 }
 
+
+
+germlineRegionOrder := ["FR1", "CDR1", "FR2", "CDR2", "FR3", "CDR3", "FR4"]
+
+germlineRegionIndex := func(region) {
+	for i, r in germlineRegionOrder {
+		if r == region {
+			return i
+		}
+	}
+	return -1
+}
+
+
+
+
+
+
+
+
+
+imputedFlankFeatures := func(assembleClonesBy) {
+	if is_undefined(assembleClonesBy) || assembleClonesBy == "CDR3" || assembleClonesBy == "VDJRegion" {
+		return []
+	}
+
+	begin := undefined
+	end := undefined
+
+	toParts := text.split(assembleClonesBy, "_TO_")
+	if len(toParts) == 2 {
+		begin = toParts[0]
+		end = toParts[1]
+	} else if text.has_prefix(assembleClonesBy, "{") && text.has_suffix(assembleClonesBy, "}") {
+		inner := assembleClonesBy[1:len(assembleClonesBy) - 1]
+		colonParts := text.split(inner, ":")
+		if len(colonParts) != 2 {
+			return []
+		}
+		begin = text.replace(colonParts[0], "Begin", "", -1)
+		end = text.replace(colonParts[1], "End", "", -1)
+	} else {
+		return []
+	}
+
+	iBegin := germlineRegionIndex(begin)
+	iEnd := germlineRegionIndex(end)
+	if iBegin == -1 || iEnd == -1 || iBegin > iEnd {
+		return []
+	}
+
+	imputed := []
+	for i := 0; i < iBegin; i++ {
+		imputed = append(imputed, germlineRegionOrder[i])
+	}
+	for i := iEnd + 1; i < len(germlineRegionOrder); i++ {
+		imputed = append(imputed, germlineRegionOrder[i])
+	}
+
+	if !(begin == "FR1" && end == "FR4") {
+		imputed = append(imputed, "VDJRegion")
+	}
+	return imputed
+}
+
 exportSpecOpsFromPreset := func(presetSpecForBack) {
 	assemblingFeature := undefined
 	if !is_undefined(presetSpecForBack.assemblingFeature) {
@@ -129,7 +194,7 @@ addSpec := func(columns, additionalSpec) {
 
 
 
-calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, exportMinQuality) {
+calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, exportMinQuality, imputeGermline, assembleClonesBy) {
 	ops := exportSpecOpsFromPreset(presetSpecForBack)
 
 	assemblingFeature := ops.assemblingFeature
@@ -137,11 +202,14 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, expor
 	cellTags := ops.cellTags
 	hasUmi := ops.hasUmi
 	splitByC := ops.splitByC
-	
+
 
 	if is_undefined(exportMinQuality) {
 		exportMinQuality = false
 	}
+	if is_undefined(imputeGermline) {
+		imputeGermline = false
+	}
 
 	isSingleCell := !is_undefined(cellTags) && len(cellTags) > 0
 	hashCellKey := isSingleCell && len(cellTags) > 1
@@ -496,10 +564,40 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, expor
 
 	annotationTypes := assemblingFeature == "CDR3" ? ["Segments"] : ["CDRs", "Segments"]
 
-	for isImputed in ( is_undefined(assemblingFeature) ? [false, true] : [false] ) {
+
+
+
+
+
+
+	imputedFeatures := imputeGermline ? imputedFlankFeatures(assembleClonesBy) : []
+	doImpute := len(imputedFeatures) > 0
+
+
+
+
+
+
+
+	if doImpute {
+		imputedSet := {}
+		for f in imputedFeatures {
+			imputedSet[f] = true
+		}
+		nonImputedFeatures := []
+		for f in features {
+			if is_undefined(imputedSet[f]) {
+				nonImputedFeatures = append(nonImputedFeatures, f)
+			}
+		}
+		features = nonImputedFeatures
+	}
+
+	for isImputed in ( doImpute ? [false, true] : [false] ) {
 		imputedU := isImputed ? "Imputed" : ""
 		imputedL := text.to_lower(imputedU)
-		for featureU in features {
+		featuresList := isImputed ? imputedFeatures : features
+		for featureU in featuresList {
 			featureL := text.to_lower(formatId(featureU))
 			for isAminoAcid in [true, false] {
 				featureInFrameU := isAminoAcid ? inFrameFeatures[featureU] : featureU
@@ -539,7 +637,7 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, expor
 								"pl7.app/vdj/isMainSequence": featureU == anchorFeature ? "true" : "false",
 								"pl7.app/vdj/imputed": string(isImputed),
 								"pl7.app/table/fontFamily": "monospace",
-								"pl7.app/label": featureInFrameU + " " + alphabetShort
+								"pl7.app/label": (isImputed ? "Imputed " : "") + featureInFrameU + " " + alphabetShort
 							})
 						}
 					} ]

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@platforma-open/milaboratories.mixcr-clonotyping-2.workflow",
-  "version": "3.26.3",
+  "version": "3.27.1",
   "description": "Tengo-based template",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "5.12.0",
+    "@platforma-sdk/workflow-tengo": "6.5.0",
     "@platforma-open/milaboratories.software-mixcr": "4.7.0-347-develop"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "3.0.2"
+    "@platforma-sdk/tengo-builder": "4.0.9"
   },
   "scripts": {
     "build": "shx rm -rf dist && pl-tengo check && pl-tengo build",

package/src/calculate-export-specs.lib.tengo

@@ -105,6 +105,71 @@ formatId := func(input) {
     return result
 }
 
+// Gene-feature regions in 5'->3' order, used to derive which regions fall outside an
+// assembled span.
+germlineRegionOrder := ["FR1", "CDR1", "FR2", "CDR2", "FR3", "CDR3", "FR4"]
+
+germlineRegionIndex := func(region) {
+	for i, r in germlineRegionOrder {
+		if r == region {
+			return i
+		}
+	}
+	return -1
+}
+
+// Derives the gene-feature regions lying OUTSIDE the assembled span (5' and 3' flanks),
+// plus the full VDJRegion, for germline imputation. Mirrors the mixcr-amplicon-alignment
+// block's parseAssemblingFeature, adapted to the "Assemble clones by" value formats:
+//   CDR3, VDJRegion    -> nothing to impute (minimal / full feature)
+//   <X>_TO_<Y>         -> span X..Y (e.g. CDR1_TO_FR4)
+//   {<X>Begin:<Y>End}  -> span X..Y (e.g. {FR1Begin:CDR3End})
+// The assembled span itself is observed, not imputed, so it never gets a duplicate
+// imputed column. Returns [] when there is nothing to impute or the value is unrecognised.
+imputedFlankFeatures := func(assembleClonesBy) {
+	if is_undefined(assembleClonesBy) || assembleClonesBy == "CDR3" || assembleClonesBy == "VDJRegion" {
+		return []
+	}
+
+	begin := undefined
+	end := undefined
+
+	toParts := text.split(assembleClonesBy, "_TO_")
+	if len(toParts) == 2 {
+		begin = toParts[0]
+		end = toParts[1]
+	} else if text.has_prefix(assembleClonesBy, "{") && text.has_suffix(assembleClonesBy, "}") {
+		inner := assembleClonesBy[1:len(assembleClonesBy) - 1]
+		colonParts := text.split(inner, ":")
+		if len(colonParts) != 2 {
+			return []
+		}
+		begin = text.replace(colonParts[0], "Begin", "", -1)
+		end = text.replace(colonParts[1], "End", "", -1)
+	} else {
+		return []
+	}
+
+	iBegin := germlineRegionIndex(begin)
+	iEnd := germlineRegionIndex(end)
+	if iBegin == -1 || iEnd == -1 || iBegin > iEnd {
+		return []
+	}
+
+	imputed := []
+	for i := 0; i < iBegin; i++ {
+		imputed = append(imputed, germlineRegionOrder[i])
+	}
+	for i := iEnd + 1; i < len(germlineRegionOrder); i++ {
+		imputed = append(imputed, germlineRegionOrder[i])
+	}
+	// VDJRegion is the full FR1..FR4; impute it unless the span already covers everything.
+	if !(begin == "FR1" && end == "FR4") {
+		imputed = append(imputed, "VDJRegion")
+	}
+	return imputed
+}
+
 exportSpecOpsFromPreset := func(presetSpecForBack) {
 	assemblingFeature := undefined
 	if !is_undefined(presetSpecForBack.assemblingFeature) {
@@ -129,7 +194,7 @@ addSpec := func(columns, additionalSpec) {
 // Ordering rules
 //   AA Sequences
 
-calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, exportMinQuality) {
+calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, exportMinQuality, imputeGermline, assembleClonesBy) {
 	ops := exportSpecOpsFromPreset(presetSpecForBack)
 
 	assemblingFeature := ops.assemblingFeature
@@ -137,11 +202,14 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, expor
 	cellTags := ops.cellTags
 	hasUmi := ops.hasUmi
 	splitByC := ops.splitByC
-	
+
 	// Default to false if not provided
 	if is_undefined(exportMinQuality) {
 		exportMinQuality = false
 	}
+	if is_undefined(imputeGermline) {
+		imputeGermline = false
+	}
 
 	isSingleCell := !is_undefined(cellTags) && len(cellTags) > 0
 	hashCellKey := isSingleCell && len(cellTags) > 1
@@ -496,10 +564,40 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, expor
 	// aaAnnotationOfSegmentsForVDJRegionInFrame
 	annotationTypes := assemblingFeature == "CDR3" ? ["Segments"] : ["CDRs", "Segments"]
 
-	for isImputed in ( is_undefined(assemblingFeature) ? [false, true] : [false] ) {
+	// Germline imputation (generic amplicon presets only): when enabled, emit additional
+	// germline-imputed sequence columns for the gene-feature regions OUTSIDE the assembled
+	// span (plus the full VDJRegion). imputedFlankFeatures returns [] for non-generic presets
+	// (assembleClonesBy undefined) and for full/minimal spans, so this never fires outside the
+	// intended scope. The clonotype key, built from the non-imputed assembling feature, is
+	// unaffected.
+	imputedFeatures := imputeGermline ? imputedFlankFeatures(assembleClonesBy) : []
+	doImpute := len(imputedFeatures) > 0
+
+	// Keep the imputed and non-imputed feature sets disjoint (as mixcr-amplicon-alignment does).
+	// A flank feature (e.g. FR1) lives in the fixed non-imputed `features` list AND in the imputed
+	// set; emitting both yields two columns with identical PColumn identity (name + domain - the
+	// `imputed` flag is annotation-only, so it does not disambiguate). The empty non-imputed column
+	// (region_not_covered) then shadows the populated germline-imputed one. So when imputing, drop
+	// those features from the non-imputed list - only the imputed column remains for the flanks.
+	if doImpute {
+		imputedSet := {}
+		for f in imputedFeatures {
+			imputedSet[f] = true
+		}
+		nonImputedFeatures := []
+		for f in features {
+			if is_undefined(imputedSet[f]) {
+				nonImputedFeatures = append(nonImputedFeatures, f)
+			}
+		}
+		features = nonImputedFeatures
+	}
+
+	for isImputed in ( doImpute ? [false, true] : [false] ) {
 		imputedU := isImputed ? "Imputed" : ""
 		imputedL := text.to_lower(imputedU)
-		for featureU in features {
+		featuresList := isImputed ? imputedFeatures : features
+		for featureU in featuresList {
 			featureL := text.to_lower(formatId(featureU))
 			for isAminoAcid in [true, false] {
 				featureInFrameU := isAminoAcid ? inFrameFeatures[featureU] : featureU
@@ -539,7 +637,7 @@ calculateExportSpecs := func(presetSpecForBack, sampleIdAxisSpec, blockId, expor
 								"pl7.app/vdj/isMainSequence": featureU == anchorFeature ? "true" : "false",
 								"pl7.app/vdj/imputed": string(isImputed),
 								"pl7.app/table/fontFamily": "monospace",
-								"pl7.app/label": featureInFrameU + " " + alphabetShort
+								"pl7.app/label": (isImputed ? "Imputed " : "") + featureInFrameU + " " + alphabetShort
 							})
 						}
 					} ]

package/src/main.tpl.tengo

@@ -134,6 +134,7 @@ wf.body(func(args) {
 				materialType: args.materialType,
 				tagPattern: args.tagPattern,
 				assembleClonesBy: args.assembleClonesBy,
+				imputeGermline: args.imputeGermline,
 				exportMinQuality: args.exportMinQuality,
 				stopCodonTypes: args.stopCodonTypes,
 				stopCodonReplacements: args.stopCodonReplacements

package/src/mixcr-analyze.tpl.tengo

@@ -86,18 +86,34 @@ self.body(func(inputs) {
 	    mixcrCmdBuilder.cpu(16)     // default CPUs number per sample
 	}
 
-	// Memory limit per process
+	// Memory limit per process.
+	// Size analyze RAM from the input reads' file size, on top of a per-analysis
+	// floor. size("reads") sums the stored (compressed) blob size of the sample's
+	// R1+R2 FASTQ via getBlobSize — a metadata read, no pre-exec — so it tracks
+	// total input volume (read count and read length) without decompressing.
+	//     mem = clamp(baseMemGiB + 4 bytes/byte * size, baseMemGiB, 256 GiB)
+	// baseMemGiB is the per-analysis floor (64, or 110/192 for contig/cell and
+	// single-cell+MiTool pipelines, which need that headroom regardless of size);
+	// the linear term scales with input volume; the 256 GiB cap bounds the largest.
+	// An "Advanced Settings" memory override takes precedence; on backends that
+	// cannot evaluate resource formulas, memFormula falls back to the baseline.
     if !is_undefined(perProcessMemGB) {
         // memory per process was set in "Advanced Settings" block directly
 		mixcrCmdBuilder.mem(string(perProcessMemGB) + "GiB")
-	} else if hasMiToolSteps {
-		// allow to use more memory for single cell analysis with mitool pre-
-		mixcrCmdBuilder.mem("192GiB")
-	} else if hasAssembleContigs || hasAssembleCells {
-		// allow to use more memory for single cell analysis
-		mixcrCmdBuilder.mem("110GiB")
 	} else {
-		mixcrCmdBuilder.mem("64GiB")
+		baseMemGiB := 64
+		if hasMiToolSteps {
+			baseMemGiB = 192
+		} else if hasAssembleContigs || hasAssembleCells {
+			baseMemGiB = 110
+		}
+		f := exec.formula
+		mixcrCmdBuilder.memFormula(
+			f.clamp(
+				f.add(f.gib(baseMemGiB), f.mul(f.size("reads"), 4)),
+				f.gib(baseMemGiB),
+				f.gib(256)),
+			{ fallback: string(baseMemGiB) + "GiB" })
 	}
 
 	if !is_undefined(limitInput) {
@@ -166,7 +182,7 @@ self.body(func(inputs) {
 		inputFile := inputMap["[]"]
 		ll.assert(!is_undefined(inputFile), "unexpected agg group structure")
 		inputFileName := "input." + fileExtension
-		mixcrCmdBuilder.addFile(inputFileName, inputFile)
+		mixcrCmdBuilder.addFile(inputFileName, inputFile, { tag: "reads" })
 		mixcrCmdBuilder.arg(inputFileName)
 	} else if inputDataMeta.keyLength == 1 {
 		ll.assert(aggregationAxesNames == ["pl7.app/sequencing/readIndex"], "unexpected aggregation axes names")
@@ -180,7 +196,7 @@ self.body(func(inputs) {
 			if (r[0] != 'R' && r[0] != "I") || (r[1] != '1' && r[1] != '2') || len(r) != 2 {
 				ll.panic("malformed read index: %v", r)
 			}
-			mixcrCmdBuilder.addFile("input_" + r + "." + fileExtension, inputFile)
+			mixcrCmdBuilder.addFile("input_" + r + "." + fileExtension, inputFile, { tag: "reads" })
 		}
 		mixcrCmdBuilder.arg("input_{{R}}." + fileExtension)
 	} else if inputDataMeta.keyLength == 2 {
@@ -199,7 +215,7 @@ self.body(func(inputs) {
 			if is_undefined(int(lane)) {
 				ll.panic("malformed lane: %v", lane)
 			}
-			mixcrCmdBuilder.addFile("input_L" + lane + "_" + r + "." + fileExtension, inputFile)
+			mixcrCmdBuilder.addFile("input_L" + lane + "_" + r + "." + fileExtension, inputFile, { tag: "reads" })
 		}
 		mixcrCmdBuilder.arg("input_L{{n}}_{{R}}." + fileExtension)
 	} else {

package/src/process.tpl.tengo

@@ -203,7 +203,7 @@ self.body(func(inputs) {
 		}
 	} ]
 
-	exportSpecs := calculateExportSpecs(presetSpecForBack, sampleIdAxisSpec, blockId, params.exportMinQuality)
+	exportSpecs := calculateExportSpecs(presetSpecForBack, sampleIdAxisSpec, blockId, params.exportMinQuality, params.imputeGermline, params.assembleClonesBy)
 
 	columnsSpecPerSample := exportSpecs.columnsSpecPerSample
 	columnsSpecPerSampleSc := exportSpecs.columnsSpecPerSampleSc

package/src/test/columns-calculate.tpl.tengo

@@ -9,7 +9,7 @@ self.defineOutputs("exportSpecs")
 self.body(func(inputs) {
 	presetSpecForBack := inputs.presetSpecForBack.getDataAsJson()
 
-	exportSpecs := calculateExportSpecs(presetSpecForBack, {}, "test", false)
+	exportSpecs := calculateExportSpecs(presetSpecForBack, {}, "test", false, false, undefined)
 	exportSpecs = maps.deepMerge(exportSpecs, {
     	axisByClonotypeKeyGen: undefined,
 		axisByScClonotypeKeyGen: undefined,