@platforma-open/milaboratories.mixcr-clonotyping-2.workflow 2.0.0

package/src/list-presets.tpl.tengo

@@ -0,0 +1,26 @@
+// list presets
+
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+file := import("@platforma-sdk/workflow-tengo:file")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+
+self.defineOutputs("presets")
+
+mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:low-memory")
+
+self.body(func(inputs) {
+	listPresets := exec.builder().
+		inUiQueue().
+		software(mixcrSw).
+		secret("MI_LICENSE", "MI_LICENSE").
+		arg("listPresetSpecificationsForUI").
+		arg("presets.json").
+		saveFile("presets.json").
+		run()
+
+	return {
+		presets: file.exportFile(listPresets.getFile("presets.json"))
+	}
+})

package/src/main.tpl.tengo

@@ -0,0 +1,159 @@
+wf := import("@platforma-sdk/workflow-tengo:workflow")
+
+render := import("@platforma-sdk/workflow-tengo:render")
+
+maps := import("@platforma-sdk/workflow-tengo:maps")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+file := import("@platforma-sdk/workflow-tengo:file")
+llPFrames := import("@platforma-sdk/workflow-tengo:pframes.ll")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+
+json := import("json")
+
+calculatePresetInfoTpl := assets.importTemplate(":calculate-preset-info")
+processTpl := assets.importTemplate(":process")
+
+wf.setPreRun(assets.importTemplate(":prerun"))
+
+wf.prepare(func(args){
+	return{
+		resolvedLibraryInput: wf.resolve(args.inputLibrary, { errIfMissing: false })
+	}
+})
+
+wf.body(func(args) {
+	// (begin) @TODO To be moved to SDK / Model
+
+	fileImports := {}
+
+	importFile := func(importHandle) {
+		fromMap := fileImports[importHandle]
+		if fromMap == undefined {
+			fImport := file.importFile(importHandle)
+			fileImports[importHandle] = fImport
+			return fImport.file
+		} else {
+			return fromMap.file
+		}
+	}
+
+	// (end)
+
+	blockId := wf.blockId().getDataAsJson()
+
+	inputRef := args.input
+
+	preset := args.preset
+
+	// file or json with preset name
+	presetResource := undefined
+
+	if preset.type == "file" {
+		presetResource = importFile(preset.file)
+	} else {
+		ll.assert(preset.type == "name", "unexpected preset type")
+		// whole json, including type
+		presetResource = smart.createJsonResource(preset)
+	}
+
+	limitInput := args.limitInput
+	isLibraryFileGzipped := false
+
+	library := false
+	species := args.species
+	libraryImportHandle := undefined
+	if !is_undefined(args.resolvedLibraryInput) {
+		library = args.resolvedLibraryInput.data
+		librarySpec := args.resolvedLibraryInput.spec
+		species = librarySpec["annotations"]["pl7.app/species"]
+	} else if !is_undefined(args.libraryFile) {
+		fImport := file.importFile(args.libraryFile)
+		libraryImportHandle = fImport.handle
+		library = fImport.file
+		species = args.customSpecies
+		isLibraryFileGzipped = args.isLibraryFileGzipped
+	}
+
+	input := wf.resolve(inputRef)
+
+	//ll.print("__THE_LOG__ " +input)
+
+	presetInfoResult := render.create(calculatePresetInfoTpl, {
+		preset: presetResource,
+		params: {
+			species: species
+		}
+	})
+
+	presetContent := presetInfoResult.output("preset", 24 * 60 * 60 * 1000)
+	presetSpecForBack := presetInfoResult.output("presetSpecForBack", 24 * 60 * 60 * 1000)
+
+	// calculating chains
+	chains := [
+		{ name: "IGH", mixcrChain: "IGH" },
+		{ name: "IGKL", mixcrChain: "IGK,IGL" },
+		{ name: "TRA", mixcrChain: "TRA" },
+		{ name: "TRB", mixcrChain: "TRB" },
+		{ name: "TRD", mixcrChain: "TRD" },
+		{ name: "TRG", mixcrChain: "TRG" } ]
+
+	runMixcr := render.createEphemeral(processTpl, {
+		inputSpec: input.getFutureInputField("spec"),
+		inputData: input.getFutureInputField("data"),
+
+		preset: presetResource,
+		presetSpecForBack: presetSpecForBack,
+		presetContent: presetContent,
+
+		library: library,
+
+		params: smart.createJsonResource({
+			species: species,
+			chains: chains,
+			limitInput: limitInput,
+			blockId: blockId,
+			presetCommonName: args.presetCommonName,
+			isLibraryFileGzipped: isLibraryFileGzipped
+		})
+	})
+
+	exports := {
+		qc: {
+			spec: runMixcr.output("qc.spec"),
+			data: runMixcr.output("qc.data")
+		},
+		reports: {
+			spec: runMixcr.output("reports.spec"),
+			data: runMixcr.output("reports.data")
+		},
+		clones: runMixcr.output("clonotypes"),
+		clns: {
+			spec: runMixcr.output("clns.spec"),
+			data: runMixcr.output("clns.data")
+		}
+	}
+
+	outputs := {
+		qc: pframes.exportColumnData(runMixcr.output("qc.data")),
+		reports: pframes.exportColumnData(runMixcr.output("reports.data")),
+		logs: runMixcr.output("logs.data"),
+		clonotypes: pframes.exportFrame(runMixcr.output("clonotypes")),
+		clns: runMixcr.output("clns.data"),
+		fileImports: smart.createMapResource(maps.mapValues(fileImports, func(im) {
+			return im.handle
+		}))
+	}
+
+	if !is_undefined(libraryImportHandle) {
+		outputs.libraryImportHandle = libraryImportHandle
+	}
+
+	return {
+		outputs: outputs,
+		exports: exports
+	}
+})

package/src/mixcr-analyze.tpl.tengo

@@ -0,0 +1,171 @@
+// mixcr analyze
+
+self := import("@platforma-sdk/workflow-tengo:tpl")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+render := import("@platforma-sdk/workflow-tengo:render")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
+pcolumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
+file := import("@platforma-sdk/workflow-tengo:file")
+times := import("times")
+
+json := import("json")
+
+self.defineOutputs("qc", "reports", "log", "clns")
+
+mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:main")
+
+progressPrefix := "[==PROGRESS==]"
+
+self.body(func(inputs) {
+	inputData := inputs[pConstants.VALUE_FIELD_NAME]
+	aggregationAxesNames := inputs[pConstants.AGGREGATION_AXES_NAMES_FIELD_NAME]
+
+	preset := inputs.preset
+	library := inputs.library
+
+	params := inputs.params
+	species := params.species
+	limitInput := params.limitInput
+	fileExtension := params.fileExtension
+	reports := params.reports
+	isLibraryFileGzipped := params.isLibraryFileGzipped
+
+	presetContent := inputs.presetContent.getDataAsJson()
+
+	if !is_map(presetContent) {
+		ll.panic("malformed presetContent %v", presetContent)
+	}
+
+	inputDataMeta := inputData.getDataAsJson()
+
+	hasAssembleContigs := false
+	hasAssembleCells := false
+	for stage in presetContent.analysisStages {
+		if stage == "assembleContigs" {
+			hasAssembleContigs = true
+		} else if stage == "assembleCells" {
+			hasAssembleCells = true
+		}
+	}
+
+	clnsFileName := "result.clns"
+	if hasAssembleContigs {
+		clnsFileName = "result.contigs.clns"
+	}
+	if hasAssembleCells {
+		clnsFileName = "result.assembledCells.clns"
+	}
+
+	mixcrCmdBuilder := exec.builder().
+		printErrStreamToStdout().
+		env("MI_PROGRESS_PREFIX", progressPrefix).
+		software(mixcrSw).
+		secret("MI_LICENSE", "MI_LICENSE").
+		arg("analyze")
+
+	if !is_undefined(limitInput) {
+		mixcrCmdBuilder.arg("--limit-input").arg(string(limitInput))
+	}
+
+	if library {
+		mixcrCmdBuilder.arg("--species").arg(species)
+		if isLibraryFileGzipped {
+			mixcrCmdBuilder.arg("--library").arg("library.json.gz").
+			addFile("library.json.gz", library)
+		} else {
+			mixcrCmdBuilder.arg("--library").arg("library.json").
+			addFile("library.json", library)
+		}
+	} else if !is_undefined(species) {
+		species = params.species
+		mixcrCmdBuilder.arg("--species").arg(species)
+	}
+
+	if smart.isResource(preset) /* file */ {
+		mixcrCmdBuilder.
+			arg("local#input_preset").
+			addFile("input_preset.yaml", preset)
+	} else {
+		ll.assert(!is_undefined(preset.name), "undefined preset name")
+		mixcrCmdBuilder.
+			arg(preset.name)
+	}
+
+	if inputDataMeta.keyLength == 0 {
+		ll.assert(aggregationAxesNames == [], "unexpected aggregation axes names")
+		inputFile := inputData.inputs()["[]"]
+		ll.assert(!is_undefined(inputFile), "unexpected agg group structure")
+		inputFileName := "input." + fileExtension
+		mixcrCmdBuilder.addFile(inputFileName, inputFile)
+		mixcrCmdBuilder.arg(inputFileName)
+	} else if inputDataMeta.keyLength == 1 {
+		ll.assert(aggregationAxesNames == ["pl7.app/sequencing/readIndex"], "unexpected aggregation axes names")
+		for sKey, inputFile in inputData.inputs() {
+			key := json.decode(sKey)
+			if len(key) != 1 {
+				ll.panic("malformed key: %v", sKey)
+			}
+			r := key[0]
+			if (r[0] != 'R' && r[0] != "I") || (r[1] != '1' && r[1] != '2') || len(r) != 2 {
+				ll.panic("malformed read index: %v", r)
+			}
+			mixcrCmdBuilder.addFile("input_" + r + "." + fileExtension, inputFile)
+		}
+		mixcrCmdBuilder.arg("input_{{R}}." + fileExtension)
+	} else if inputDataMeta.keyLength == 2 {
+		ll.assert(aggregationAxesNames == ["pl7.app/sequencing/lane", "pl7.app/sequencing/readIndex"], "unexpected aggregation axes names")
+		for sKey, inputFile in inputData.inputs() {
+			key := json.decode(sKey)
+			if len(key) != 2 {
+				ll.panic("malformed key: %v", sKey)
+			}
+			lane := key[0]
+			r := key[1]
+			if (r[0] != 'R' && r[0] != "I") || (r[1] != '1' && r[1] != '2') || len(r) != 2 {
+				ll.panic("malformed read index: %v", r)
+			}
+			if is_undefined(int(lane)) {
+				ll.panic("malformed lane: %v", lane)
+			}
+			mixcrCmdBuilder.addFile("input_L" + lane + "_" + r + "." + fileExtension, inputFile)
+		}
+		mixcrCmdBuilder.arg("input_L{{n}}_{{R}}." + fileExtension)
+	} else {
+		ll.panic("too many axes / not supported")
+	}
+
+	mixcrCmdBuilder.arg("result")
+
+	mixcrCmdBuilder.saveFile("result.qc.json")
+	mixcrCmdBuilder.saveFile(clnsFileName)
+
+	for report in reports {
+		mixcrCmdBuilder.saveFile(report.fileJson)
+		mixcrCmdBuilder.saveFile(report.fileTxt)
+	}
+
+	mixcrCmdBuilder = mixcrCmdBuilder.cache(48 * times.hour)
+	mixcrCmd := mixcrCmdBuilder.run()
+
+	// collecting results
+
+	reportsMap := pcolumn.resourceMapBuilder( /* keyLength */ 2 )
+	for report in reports {
+		reportsMap.add([report.id, "json"], mixcrCmd.getFile(report.fileJson))
+		reportsMap.add([report.id, "txt"], mixcrCmd.getFile(report.fileTxt))
+	}
+
+	result := {
+		qc: mixcrCmd.getFile("result.qc.json"),
+		log: mixcrCmd.getStdoutStream(),
+		reports: reportsMap.build(),
+		clns: mixcrCmd.getFile(clnsFileName)
+	}
+
+	return result
+})

package/src/mixcr-export.tpl.tengo

@@ -0,0 +1,77 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+
+json := import("json")
+
+self.defineOutputs("tsv")
+
+mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:low-memory")
+ptransformSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
+
+self.body(func(inputs) {
+	clnsFile := inputs[pConstants.VALUE_FIELD_NAME]
+
+	params := inputs.params
+	chains := params.chains
+	exportArgs := params.exportArgs
+
+	clonotypeKeyColumns := params.clonotypeKeyColumns
+
+	// Exporting clones from clns file
+
+	mixcrCmdBuilder := exec.builder().
+		printErrStreamToStdout().
+		software(mixcrSw).
+		secret("MI_LICENSE", "MI_LICENSE").
+		arg("exportClones").
+		arg("--dont-split-files").
+		arg("--chains").arg(chains)
+
+	for argGrp in exportArgs {
+		for arg in argGrp {
+			mixcrCmdBuilder.arg(arg)
+		}
+	}
+
+	mixcrCmd := mixcrCmdBuilder.
+		arg("clones.clns").
+		addFile("clones.clns", clnsFile).
+		arg("clones.tsv").
+		saveFile("clones.tsv").
+		run()
+
+	unprocessedTsv := mixcrCmd.getFile("clones.tsv")
+
+	if is_undefined(clonotypeKeyColumns) {
+		return {
+			tsv: unprocessedTsv
+		}
+	} else {
+		// Adding clonotypeKey column
+		pWorkflow := {
+			steps: [ {
+				type: "combine_columns_as_json",
+				src: clonotypeKeyColumns,
+				dst: "clonotypeKey"
+			} ]
+		}
+
+		aggregateCmd := exec.builder().
+			printErrStreamToStdout().
+			software(ptransformSw).
+			arg("--workflow").arg("wf.json").
+			writeFile("wf.json", json.encode(pWorkflow)).
+			arg("input.tsv").addFile("input.tsv", unprocessedTsv).
+			arg("output.tsv").saveFile("output.tsv").
+			run()
+
+		processedTsv := aggregateCmd.getFile("output.tsv")
+
+		return {
+			tsv: processedTsv
+		}
+	}
+})

package/src/prerun.tpl.tengo

@@ -0,0 +1,70 @@
+// pre-run
+
+wf := import("@platforma-sdk/workflow-tengo:workflow")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+render := import("@platforma-sdk/workflow-tengo:render")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+file := import("@platforma-sdk/workflow-tengo:file")
+
+listPresetsTpl := assets.importTemplate(":list-presets")
+calculatePresetInfoTpl := assets.importTemplate(":calculate-preset-info")
+
+wf.body(func(args) {
+
+	// (begin) @TODO To be moved to SDK / Model
+
+	fileImports := {}
+
+	importFile := func(importHandle) {
+		fromMap := fileImports[importHandle]
+		if fromMap == undefined {
+			fImport := file.importFile(importHandle)
+			fileImports[importHandle] = fImport
+			return fImport.file
+		} else {
+			return fromMap.file
+		}
+	}
+
+	// (end)
+
+	outputs := {}
+
+	listPresets := render.create(listPresetsTpl, {})
+	outputs.presets = listPresets.output("presets", 24 * 60 * 60 * 1000)
+
+	if !is_undefined(args.preset) {
+
+		preset := args.preset
+
+		// file or json with preset name
+		presetResource := undefined
+
+		if preset.type == "file" {
+			//presetResource = importFile(preset.file)
+			presetResource = undefined
+		} else {
+			ll.assert(preset.type == "name", "unexpected preset type")
+			// whole json, including type
+			presetResource = smart.createJsonResource(preset)
+			presetInfoResult := render.create(calculatePresetInfoTpl, {
+				preset: presetResource,
+				params: {
+					species: args.species
+				}
+			})
+			outputs.preset = presetInfoResult.output("presetSpecForBack", 24 * 60 * 60 * 1000)
+		}
+	}
+
+	outputs.fileImports = smart.createMapResource(maps.mapValues(fileImports, func(im) {
+		return im.handle
+	}))
+
+	return {
+		outputs: outputs,
+		exports: {}
+	}
+})