@platforma-open/milaboratories.mixcr-library-builder.workflow 1.3.1 → 2.0.0

package/.turbo/turbo-build.log

@@ -1,13 +1,19 @@
  WARN  Issue while reading "/home/runner/work/mixcr-library-builder/mixcr-library-builder/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.mixcr-library-builder.workflow@1.3.1 build /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow
+> @platforma-open/milaboratories.mixcr-library-builder.workflow@2.0.0 build /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
+Processing "src/fasta-generation.tpl.tengo"...
+Processing "src/fasta-to-table.tpl.tengo"...
 Processing "src/formatText.lib.tengo"...
+Processing "src/json-merging.tpl.tengo"...
 Processing "src/main.tpl.tengo"...
 No syntax errors found.
   info: Compiling 'dist'...
   info:   - writing /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow/dist/tengo/lib/formatText.lib.tengo
+  info:   - writing /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow/dist/tengo/tpl/fasta-generation.plj.gz
+  info:   - writing /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow/dist/tengo/tpl/fasta-to-table.plj.gz
+  info:   - writing /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow/dist/tengo/tpl/json-merging.plj.gz
   info:   - writing /home/runner/work/mixcr-library-builder/mixcr-library-builder/workflow/dist/tengo/tpl/main.plj.gz
   info: Template Pack build done.
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @platforma-open/milaboratories.mixcr-library-builder.workflow
 
+## 2.0.0
+
+### Major Changes
+
+- 5a071f6: Multiple chains reference builder, updated dependecies
+
+### Patch Changes
+
+- Updated dependencies [5a071f6]
+  - @platforma-open/milaboratories.mixcr-library-builder.software@2.0.0
+
+## 1.4.0
+
+### Minor Changes
+
+- fc66573: upport batch system
+
 ## 1.3.1
 
 ### Patch Changes

package/dist/index.cjs

@@ -1,3 +1,6 @@
 module.exports = { Templates: {
+  'fasta-generation': { type: 'from-file', path: require.resolve('./tengo/tpl/fasta-generation.plj.gz') },
+  'fasta-to-table': { type: 'from-file', path: require.resolve('./tengo/tpl/fasta-to-table.plj.gz') },
+  'json-merging': { type: 'from-file', path: require.resolve('./tengo/tpl/json-merging.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "main";
+declare type TplName = "fasta-generation" | "fasta-to-table" | "json-merging" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,4 +1,7 @@
 import { resolve } from 'node:path';
 export const Templates = {
+  'fasta-generation': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/fasta-generation.plj.gz') },
+  'fasta-to-table': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/fasta-to-table.plj.gz') },
+  'json-merging': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/json-merging.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/package.json

@@ -1,14 +1,18 @@
 {
   "name": "@platforma-open/milaboratories.mixcr-library-builder.workflow",
-  "version": "1.3.1",
+  "version": "2.0.0",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "^4.7.1"
+    "@platforma-sdk/workflow-tengo": "^4.9.3",
+    "@platforma-open/milaboratories.software-mixcr": "4.7.0-139-develop",
+    "@platforma-open/milaboratories.software-repseqio": "^2.5.0-13-master",
+    "@platforma-open/milaboratories.mixcr-library-builder.software": "2.0.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.1.7",
-    "@platforma-open/milaboratories.software-mixcr": "4.7.0-139-develop"
+    "@platforma-sdk/tengo-builder": "^2.1.12",
+    "@platforma-sdk/test": "^1.39.7",
+    "vitest": "~2.1.8"
   },
   "scripts": {
     "build": "rm -rf dist && pl-tengo check && pl-tengo build",

package/src/fasta-generation.tpl.tengo

@@ -0,0 +1,59 @@
+exec := import("@platforma-sdk/workflow-tengo:exec")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+file := import("@platforma-sdk/workflow-tengo:file")
+text := import("text")
+
+repseqioSw := assets.importSoftware("@platforma-open/milaboratories.software-repseqio:main")
+
+self.defineOutputs("fasta")
+
+self.body(func(inputs) {
+    libraryMap := inputs.libraryMap
+    config := inputs.config
+    coveredRegion := ""
+
+    fastaMap := {}
+
+    for name, library in libraryMap {
+        for chain, segments in config {
+            library_chain_temp := text.split(name, "_")[1]
+            library_chain := text.split(library_chain_temp, ".")[0]
+            if library_chain == chain {
+                for segment, segmentConfig in segments {
+                // Skip C segment as it doesn't need FASTA generation
+                    if segment == "C" {
+                        continue
+                    }
+                    
+                    // Check if this segment has valid configuration
+                    if !is_undefined(segmentConfig.builtInSpecies) || !is_undefined(segmentConfig.fastaFile) {
+                        if segment == "V" {
+                            coveredRegion = segmentConfig.vRegionType
+                        } else if segment == "J" {
+                            coveredRegion = "JRegion"
+                        } else if segment == "D" {
+                            coveredRegion = "DRegion"
+                        }
+                        
+                        // Execute repseqio command only for valid segments
+                        repseqioCmdBuilder := exec.builder().
+                            software(repseqioSw).
+                            arg("fasta").
+                            arg("-g").arg(coveredRegion).
+                            arg(name).addFile(name, library).
+                            arg(segment + "_" + chain + ".fasta").
+                            saveFile(segment + "_" + chain + ".fasta")
+                        repseqioCmd := repseqioCmdBuilder.run()
+                        fasta := repseqioCmd.getFile(segment + "_" + chain + ".fasta")
+                        fastaMap[segment + "_" + chain + ".fasta"] = fasta
+                    }
+                }
+            }
+        }
+    }
+    return {
+        fasta: fastaMap
+    }
+})

package/src/fasta-to-table.tpl.tengo

@@ -0,0 +1,95 @@
+exec := import("@platforma-sdk/workflow-tengo:exec")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
+
+
+
+fastaToTableSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-library-builder.software:fasta_to_table")
+
+self.defineOutputs("fastaTable")
+
+self.body(func(args) {
+    fastaMap := args.fasta
+
+    cmdBuilder := exec.builder().
+        software(fastaToTableSw)
+
+    // The python script takes an input directory. We add all fasta files to the root of the exec env.
+    for name, fastaFile in fastaMap {
+        cmdBuilder.addFile(name, fastaFile)
+    }
+
+    // Pass the root directory "." as the input_dir argument.
+    cmd := cmdBuilder.arg(".").
+    saveFile("combined_table.tsv").
+    run()
+
+    fastaTable := cmd.getFile("combined_table.tsv")
+
+    // Create column specifications for the fastaTable
+    fastaTableSpec := {
+        axes: [{
+                column: "chain",
+                spec: {
+                    name: "pl7.app/vdj/chain",
+                    type: "String",
+                    annotations: {
+                        "pl7.app/label": "Chain"
+                    }
+                }
+            },
+            {
+                column: "segment",
+                spec: { 
+                    name: "pl7.app/vdj/segment",
+                    type: "String",
+                    annotations: {
+                        "pl7.app/label": "Segment"
+                    }
+                }
+            },
+            {
+                column: "gene_name",
+                spec: {
+                    name: "pl7.app/vdj/gene_name",
+                    type: "String",
+                    annotations: {
+                        "pl7.app/label": "Gene Name"
+                    }
+                }
+            }],
+            columns: [{
+                column: "sequence",
+                spec: {
+                    name: "pl7.app/vdj/sequence",
+                    valueType: "String",
+                    annotations: {
+                        "pl7.app/label": "AA Sequence"
+                    }
+                }
+            }
+        ],
+        storageFormat: "Binary",
+		partitionKeyLength: 2
+    }
+
+    result := xsv.importFile(
+		fastaTable,
+		"tsv",
+		fastaTableSpec,
+		{ splitDataAndSpec: true }
+	)
+
+    pf := pframes.pFrameBuilder()
+	for id, v in result {
+		pf.add(id, v.spec, v.data)
+	}
+	pf = pf.build()
+
+    return {
+        fastaTable: pframes.exportFrame(pf)
+    }
+}) 

package/src/json-merging.tpl.tengo

@@ -0,0 +1,32 @@
+exec := import("@platforma-sdk/workflow-tengo:exec")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+file := import("@platforma-sdk/workflow-tengo:file")
+
+mergeSw := assets.importSoftware("@platforma-open/milaboratories.mixcr-library-builder.software:main")
+
+self.defineOutputs("library")
+
+self.body(func(args) {
+    libraryMap := args.libraryMap
+	//fastaMap := args.fasta
+	
+	mergeCmdBuilder := exec.builder().
+		software(mergeSw).
+		arg("-o").arg("merged_library.json").
+		saveFile("merged_library.json").
+        arg("-i")
+	
+	for name, library in libraryMap {
+		mergeCmdBuilder.arg(name).
+			addFile(name, library)
+	}
+	
+    mergeCmd := mergeCmdBuilder.run()
+	finalLibrary := mergeCmd.getFile("merged_library.json")
+
+	return {
+		library: finalLibrary
+	}
+})

package/src/main.tpl.tengo

@@ -1,96 +1,159 @@
-// "hello world"
 wf := import("@platforma-sdk/workflow-tengo:workflow")
+render := import("@platforma-sdk/workflow-tengo:render")
 exec := import("@platforma-sdk/workflow-tengo:exec")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 file := import("@platforma-sdk/workflow-tengo:file")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+pcolumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
 times := import("times")
 json := import("json")
 rand := import("rand")
 text := import("text")
 
 mixcrSw := assets.importSoftware("@platforma-open/milaboratories.software-mixcr:main")
-
+jsonMergingTpl := assets.importTemplate(":json-merging")
+fastaGenerationTpl := assets.importTemplate(":fasta-generation")
+fatsaToTableTpl := assets.importTemplate(":fasta-to-table")
 toCamelCase := import(":formatText")
 
-wf.body(func(args) {
+chainInfos := {
+	"IGH": "IGHeavy",
+	"IGL": "IGLight",
+	"IGK": "IGLight",
+	"TRB": "TCRBeta",
+	"TRA": "TCRAlpha",
+	"TRG": "TCRGamma",
+	"TRD": "TCRDelta"
+}
 
+wf.body(func(args) {
 	blockId := wf.blockId().getDataAsJson()
-
-	if is_undefined(args.vFastaFile) && is_undefined(args.vSpecies) {
-		ll.panic("expected to have either `fastaFile` or `built in`; provided no any")
-	}
-
-	if is_undefined(args.jFastaFile) && is_undefined(args.jSpecies) {
-	 	ll.panic("expected to have either `fastaFile` or `built in`; provided no any")
-	}
-
-	outputs := {}
-
+	chainConfigs := args.chainConfigs
 	species := args.species
 	cmdSpecies := toCamelCase.toCamelCase(species) + "_custom"
-	chain := args.chain
-
-	libraryBuilderCmdBuilder := undefined
-
 	taxonId := string(rand.intn(9000) + 1000)
-	libraryBuilderCmdBuilder = exec.builder().
-		software(mixcrSw).
-		secret("MI_LICENSE", "MI_LICENSE").
-	 	arg("buildLibrary").
-	 	arg("--species").arg(cmdSpecies).
-	 	arg("--chain").arg(chain).
-	 	arg("--taxon-id").arg(taxonId)
-	if !is_undefined(args.vFastaFile) {
-	 	vImport := file.importFile(args.vFastaFile)
-		outputs.vImportHandle = vImport.handle
-	 	libraryBuilderCmdBuilder.arg("--v-gene-feature").arg("VRegion").
-	 	arg("--v-genes-from-fasta").arg("v.fasta").
-	 	addFile("v.fasta", vImport.file)
-	} else {
-	 	libraryBuilderCmdBuilder.arg("--v-genes-from-species").arg(args.vSpecies)
+	outputs := {}
+	fileImports := {}
+	libraryMap := {}
+	debugOutputMap := {}
+	fileImportsProgress := {}
+	chains := []
+	fivePrimePrimer := args.fivePrimePrimer
+
+	for chain, chainConfig in chainConfigs {
+		libraryBuilderCmdBuilder := exec.builder().
+			software(mixcrSw).
+			secret("MI_LICENSE", "MI_LICENSE").
+			arg("buildLibrary").
+			arg("--species").arg(cmdSpecies).
+			arg("--chain").arg(chain).
+			arg("--taxon-id").arg(taxonId).
+			arg("--debug")
+
+		if chainConfig["V"].sourceType == "built-in" {
+			libraryBuilderCmdBuilder.arg("--v-genes-from-species").arg(chainConfig["V"].builtInSpecies)
+		} else if chainConfig["V"].sourceType == "fasta" {
+			vImport := file.importFile(chainConfig["V"].fastaFile)
+			fileImports[chainConfig["V"].fastaFile] = vImport
+			fileImportsProgress[chainConfig["V"].fastaFile] = vImport.handle
+			libraryBuilderCmdBuilder.arg("--v-gene-feature").arg("VRegion").
+				arg("--v-genes-from-fasta").arg("v.fasta").
+				addFile("v.fasta", vImport.file)
+		}
+
+		if chainConfig["J"].sourceType == "built-in" {
+			libraryBuilderCmdBuilder.arg("--j-genes-from-species").arg(chainConfig["J"].builtInSpecies)
+		} else if chainConfig["J"].sourceType == "fasta" {
+			jImport := file.importFile(chainConfig["J"].fastaFile)
+			fileImports[chainConfig["J"].fastaFile] = jImport
+			fileImportsProgress[chainConfig["J"].fastaFile] = jImport.handle
+			libraryBuilderCmdBuilder.arg("--j-genes-from-fasta").arg("j.fasta").
+				addFile("j.fasta", jImport.file)
+		}
+
+		if chainConfig["D"].builtInSpecies != undefined || chainConfig["D"].fastaFile != undefined {
+			if chainConfig["D"].sourceType == "built-in" {
+				libraryBuilderCmdBuilder.arg("--d-genes-from-species").arg(chainConfig["D"].builtInSpecies)
+			} else if chainConfig["D"].sourceType == "fasta" {
+				dImport := file.importFile(chainConfig["D"].fastaFile)
+				fileImports[chainConfig["D"].fastaFile] = dImport
+				fileImportsProgress[chainConfig["D"].fastaFile] = dImport.handle
+				libraryBuilderCmdBuilder.arg("--d-genes-from-fasta").arg("d.fasta").
+					addFile("d.fasta", dImport.file)
+			}
+		}
+
+		if chainConfig["C"].builtInSpecies != undefined || chainConfig["C"].fastaFile != undefined {
+			if chainConfig["C"].sourceType == "built-in" {
+				libraryBuilderCmdBuilder.arg("--c-genes-from-species").arg(chainConfig["C"].builtInSpecies)
+			} else if chainConfig["C"].sourceType == "fasta" {
+				cImport := file.importFile(chainConfig["C"].fastaFile)
+				fileImports[chainConfig["C"].fastaFile] = cImport
+				fileImportsProgress[chainConfig["C"].fastaFile] = cImport.handle
+				libraryBuilderCmdBuilder.arg("--c-genes-from-fasta").arg("c.fasta").
+					addFile("c.fasta", cImport.file)
+			}
+		}
+
+		libraryBuilderCmdBuilder.arg("library_" + chain + ".json").
+			saveFile("library_" + chain + ".json").
+			printErrStreamToStdout()
+		libraryBuilderCmd := libraryBuilderCmdBuilder.run()
+		
+		debugOutput := libraryBuilderCmd.getStdoutStream()
+		library := libraryBuilderCmd.getFile("library_" + chain + ".json")
+		libraryMap["library_" + chain + ".json"] = library
+		debugOutputMap[chain] = debugOutput
+		chains = append(chains, chainInfos[chain])
 	}
 
-	if !is_undefined(args.jFastaFile) {
-	 	jImport := file.importFile(args.jFastaFile)
-		outputs.jImportHandle = jImport.handle
-	 	libraryBuilderCmdBuilder.arg("--j-genes-from-fasta").arg("j.fasta").
-	 		addFile("j.fasta", jImport.file)
-	} else {
-	 	libraryBuilderCmdBuilder.arg("--j-genes-from-species").arg(args.jSpecies)
+	seen := {}
+	chains_unique := []
+	for chain in chains {
+		if !seen[chain] {
+			seen[chain] = true
+			chains_unique = append(chains_unique, chain)
+		}
 	}
 
-	if !is_undefined(args.dFastaFile) {
-	 	dImport := file.importFile(args.dFastaFile)
-		outputs.dImportHandle = dImport.handle
-	 	libraryBuilderCmdBuilder.arg("--d-genes-from-fasta").arg("d.fasta").
-	 	addFile("d.fasta", dImport.file)
-	} else if !is_undefined(args.dSpecies) {
-	 	libraryBuilderCmdBuilder.arg("--d-genes-from-species").arg(args.dSpecies)
+	// Create logs map using pcolumn.resourceMapBuilder
+	debugOutputMapBuilder := pcolumn.resourceMapBuilder(/* keyLength */ 1)
+	for chain, debugOutput in debugOutputMap {
+		debugOutputMapBuilder.add([chain], debugOutput)
 	}
 
-	if !is_undefined(args.cFastaFile) {
-		cImport := file.importFile(args.cFastaFile)
-		outputs.cImportHandle = cImport.handle
-	 	libraryBuilderCmdBuilder.arg("--c-genes-from-fasta").arg("c.fasta").
-	 	addFile("c.fasta", cImport.file)
-	} else if !is_undefined(args.cSpecies) {
-	 	libraryBuilderCmdBuilder.arg("--c-genes-from-species").arg(args.cSpecies)
-	}
+	outputs.debugOutput = debugOutputMapBuilder.build()
+
+	outputs.fileImports = smart.createMapResource(maps.mapValues(fileImportsProgress, func(handle) {
+		return handle
+	}))
+
+	fastaGeneration := render.create(fastaGenerationTpl, {
+		libraryMap: libraryMap,
+		config: chainConfigs	
+		})
+	fasta := fastaGeneration.output("fasta", 24 * 60 * 60 * 1000)
 
-	libraryBuilderCmdBuilder.arg("library.json").
-		saveFile("library.json").
-		printErrStreamToStdout()
-	libraryBuilderCmd := libraryBuilderCmdBuilder.run()
+	fastaToTable := render.create(fatsaToTableTpl, {
+		fasta: fasta
+	})
+	fastaTable := fastaToTable.output("fastaTable", 24 * 60 * 60 * 1000)
 
-	debugOutput := libraryBuilderCmd.getStdoutStream()
-	library := libraryBuilderCmd.getFile("library.json")
+	mergeResults := render.create(jsonMergingTpl, {
+		libraryMap: libraryMap,
+		fastaTable: fastaTable
+		})
 
-	outputs.debugOutput = debugOutput
+	finalLibrary := mergeResults.output("library", 24 * 60 * 60 * 1000)
+
+	outputs.fastaTable = fastaTable
 
 	exports := {
-		library : {
-			data: library,
+		library: {
+			data: finalLibrary,
 			spec: {
 				kind: "File",
 				name: "pl7.app/vdj/library",
@@ -99,15 +162,15 @@ wf.body(func(args) {
 				},
 				annotations: {
 					"pl7.app/species": cmdSpecies,
-					"pl7.app/vdj/chain": chain,
+					"pl7.app/vdj/chain": string(chains),
 					"pl7.app/vdj/isLibrary": "true",
 					"pl7.app/vdj/libraryFormat": "repseqio.json",
-					"pl7.app/label": text.split(species, "_custom")[0] + " " + chain + " library"
+					"pl7.app/label": text.split(species, "_custom")[0] + " library"
 				}
 			}
-	 	}
+		}
 	}
-
+	
 	return {
 		outputs: outputs,
 		exports: exports