@platforma-open/milaboratories.top-antibodies.workflow 1.10.5 → 1.11.0

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.top-antibodies.workflow@1.10.5 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
+> @platforma-open/milaboratories.top-antibodies.workflow@1.11.0 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/filter-and-sample.tpl.tengo"...

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @platforma-open/milaboratories.top-antibodies.workflow
 
+## 1.11.0
+
+### Minor Changes
+
+- 67443d9: Move all calculations to prerun
+
 ## 1.10.5
 
 ### Patch Changes

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@platforma-open/milaboratories.top-antibodies.workflow",
-  "version": "1.10.5",
+  "version": "1.11.0",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
     "@platforma-sdk/workflow-tengo": "^5.2.2",
-    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.4.3",
     "@platforma-open/milaboratories.top-antibodies.umap": "1.1.3",
-    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.4.3"
+    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.4.3",
+    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.4.3"
   },
   "devDependencies": {
     "@platforma-sdk/tengo-builder": "^2.1.21",

package/src/main.tpl.tengo

@@ -1,326 +1,13 @@
-// light block with no workflow
 wf := import("@platforma-sdk/workflow-tengo:workflow")
-exec := import("@platforma-sdk/workflow-tengo:exec")
 assets:= import("@platforma-sdk/workflow-tengo:assets")
-xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
-pframes := import("@platforma-sdk/workflow-tengo:pframes")
-spectratypeConv := import(":pf-spectratype-conv")
-vjUsageConv := import(":pf-vj-usage-conv")
-slices := import("@platforma-sdk/workflow-tengo:slices")
-render := import("@platforma-sdk/workflow-tengo:render")
-
-filterAndSampleTpl := assets.importTemplate(":filter-and-sample")
 
 // Set prerun template for clonotype filtering
 wf.setPreRun(assets.importTemplate(":prerun"))
 
-wf.prepare(func(args){
-	// We need a table with cluster ID (optional) | clonotype id | selected ranking columns
-    bundleBuilder := wf.createPBundleBuilder()
-    bundleBuilder.ignoreMissingDomains() // to make query work for both bulk and single cell data
-    bundleBuilder.addAnchor("main", args.inputAnchor) 
-    
-    if len(args.rankingOrder) > 0 {
-        for col in args.rankingOrder {
-            bundleBuilder.addAnchor(col.value.anchorName, col.value.anchorRef)
-            bundleBuilder.addSingle(col.value.column)
-        }
-    } else {
-        bundleBuilder.addAnchor(args.rankingOrderDefault.value.anchorName, 
-                                args.rankingOrderDefault.value.anchorRef)
-        bundleBuilder.addSingle(args.rankingOrderDefault.value.column)
-    }
-
-    // Load filter columns
-    if len(args.filters) > 0 {
-        for filter in args.filters {
-            if filter.value != undefined {
-                bundleBuilder.addAnchor(filter.value.anchorName, filter.value.anchorRef)
-                bundleBuilder.addSingle(filter.value.column)
-            }
-        }
-    }
-    
-
-    // Add linker column
-    bundleBuilder.addMulti({
-        axes: [{ anchor: "main", idx: 1 }], // this will do partial axes match (unlike in the model)
-        annotations: { "pl7.app/isLinkerColumn": "true" },
-        partialAxesMatch: true
-    }, "linkers")
-
-    // Add cluster size columns from clustering blocks
-    bundleBuilder.addMulti({
-        name: "pl7.app/vdj/clustering/clusterSize",
-        partialAxesMatch: true
-    }, "clusterSizes")
-
-    // Add CDR3 sequences
-	bundleBuilder.addMulti({
-		axes: [{ anchor: "main", idx: 1 }], // Clonotype axis
-		name: "pl7.app/vdj/sequence",
-		domain: {
-			"pl7.app/alphabet": "aminoacid",
-			"pl7.app/vdj/feature": "CDR3"    // Specify CDR3 feature
-		}
-	}, "cdr3Sequences") // New collection name for CDR3 sequences
-
-	// Add V gene
-	bundleBuilder.addMulti({
-		axes: [{ anchor: "main", idx: 1 }], // Clonotype axis
-		name: "pl7.app/vdj/geneHit",
-		domain: {
-			"pl7.app/vdj/reference": "VGene"
-		}
-	}, "VGenes")
-
-	// Add J gene
-	bundleBuilder.addMulti({
-		axes: [{ anchor: "main", idx: 1 }], // Clonotype axis
-		name: "pl7.app/vdj/geneHit",
-		domain: {
-			"pl7.app/vdj/reference": "JGene"
-		}
-	}, "JGenes")
-    
+wf.body(func(args) {
     return {
-        columns: bundleBuilder.build()
+        outputs: {},
+        exports: {}
     }
 })
 
-wf.body(func(args) {
-
-	// Input arguments
-	columns := args.columns
-	datasetSpec := columns.getSpec(args.inputAnchor)
-	topClonotypes := args.topClonotypes
-
-	// Needed conditional variable
-	isSingleCell := datasetSpec.axesSpec[1].name == "pl7.app/vdj/scClonotypeKey"
-	
-	// Output container
-	outputs := {}
-
-	// Build clonotype table csv for filtering script
-	cloneTable := pframes.csvFileBuilder() 
-	cloneTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
-
-	// Add filter columns to table
-	// Filter structure {id: UI id, value: AnchoredColumnId, filter: Filter criteria, isExpanded: boolean (UI state)}
-	// AnchoredColumnId {anchorRef: PlRef, anchorName: string, column: SUniversalPColumnId (unique column id)}
-	addedAxes := [] // Keep track of axes that are added to the table to prevent duplicates
-	filterMap := {} // Map column headers to filter criteria
-	rankingMap := {} // Map column headers to ranking order (increasing/decreasing)
-	if len(args.filters) > 0 {
-		for i, filter in args.filters {
-			if filter.value != undefined {
-				// Columns added here might also be in ranking list, so we add default IDs
-				cloneTable.add(columns.getColumn(filter.value.column), 
-								{header: "Filter_" + string(i), id: "filter_" + string(i)})
-				// Store reference value and filter type associated to this column
-				filterMap["Filter_" + string(i)] = filter.filter
-			
-				// If column does not have main anchor axis we have to include theirs (columns coming from clustering blocks for example)
-				colsSpec := columns.getSpec(filter.value.column)
-				axesNames := slices.map(colsSpec.axesSpec, func (a) {return a.name})
-				if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
-					for na, ax in colsSpec.axesSpec {
-						if ax.name != datasetSpec.axesSpec[1].name {
-							cloneTable.setAxisHeader(ax.name, "cluster_" + string(i) + string(na))
-							addedAxes = append(addedAxes, ax.name)
-						}
-					}
-				}	
-			}
-		}
-	}
-
-	// Add ranking columns to table
-	if len(args.rankingOrder) > 0 {
-		for i, col in args.rankingOrder {
-			cloneTable.add(columns.getColumn(col.value.column), {header: "Col" + string(i)})
-			// Store ranking order for this column
-			rankingMap["Col" + string(i)] = col.rankingOrder
-		
-			// If column does not have main anchor axis we have to include theirs (columns coming from clustering blocks for example)
-			colsSpec := columns.getSpec(col.value.column)
-			axesNames := slices.map(colsSpec.axesSpec, func (a) {return a.name})
-			if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
-				for na, ax in colsSpec.axesSpec {
-					if ax.name != datasetSpec.axesSpec[1].name && !slices.hasElement(addedAxes, ax.name) { // Prevent duplicates
-						cloneTable.setAxisHeader(ax.name, "cluster_" + string(i) + string(na))
-					}
-				}
-			}	
-		}
-	} else {
-		i := 0
-		cloneTable.add(columns.getColumn(args.rankingOrderDefault.value.column), {header: "Col" + string(i)})
-		// Store default ranking order
-		rankingMap["Col" + string(i)] = args.rankingOrderDefault.rankingOrder
-	
-		// If column does not have main anchor axis we have to include theirs
-		colsSpec := columns.getSpec(args.rankingOrderDefault.value.column)
-		axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
-		if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
-			for na, ax in colsSpec.axesSpec {
-				if ax.name != datasetSpec.axesSpec[1].name {
-					cloneTable.setAxisHeader(ax.name, "cluster_" + string(i) + string(na))
-				}
-			}
-		}	
-	}
-
-	// Add linker columns when needed
-	linkerAxisSpec := {} // Map cluster axis names to specs. Is it needed????
-	if len(columns.getColumns("linkers")) > 0 {
-		for i, col in columns.getColumns("linkers") {
-			if datasetSpec.axesSpec[1].name == col.spec.axesSpec[1].name {
-				cloneTable.add(col, {header: "linker." + string(i)})
-				cloneTable.setAxisHeader(col.spec.axesSpec[0].name, "cluster_" + string(i))
-				linkerAxisSpec["cluster_" + string(i)] = col.spec.axesSpec[0]
-			} else if datasetSpec.axesSpec[1].name == col.spec.axesSpec[0].name {
-				cloneTable.add(col, {header: "linker." + string(i)})
-				cloneTable.setAxisHeader(col.spec.axesSpec[1].name, "cluster_" + string(i))
-				linkerAxisSpec["cluster_" + string(i)] = col.spec.axesSpec[1]
-			}
-		}
-	}
-
-	// Add cluster size columns if available
-	if len(columns.getColumns("clusterSizes")) > 0 {
-		for i, col in columns.getColumns("clusterSizes") {
-			cloneTable.add(col, {header: "clusterSize." + string(i)})
-			// Add the cluster axis header
-			for axisIdx, axis in col.spec.axesSpec {
-				if axis.name != datasetSpec.axesSpec[1].name {
-					cloneTable.setAxisHeader(axis.name, "clusterAxis_" + string(i) + "_" + string(axisIdx))
-				}
-			}
-		}
-	}
-
-	cloneTable.mem("16GiB")
-	cloneTable.cpu(1)
-	cloneTable = cloneTable.build()
-
-	// Use render.create to call the filtering and sampling clonotypes template
-	filterSampleResult := render.create(filterAndSampleTpl, {
-		inputAnchor: args.inputAnchor,
-		cloneTable: cloneTable,
-		rankingOrder: args.rankingOrder,
-		rankingOrderDefault: args.rankingOrderDefault,
-		filters: args.filters,
-		filterMap: filterMap,
-		rankingMap: rankingMap,
-		datasetSpec: datasetSpec,
-		topClonotypes: args.topClonotypes
-	})
-	
-	// Get the filtered and sampled clonotypes P-frame and CSV from the template result
-	finalClonotypesCsv := filterSampleResult.output("finalClonotypesCsv", 24 * 60 * 60 * 1000)
-	// outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
-	
-	////////// CDR3 Length Calculation //////////
-	
-	cdr3SeqTable := pframes.tsvFileBuilder()
-	cdr3SeqTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
-
-    // Must deal with multiple CDR3 sequences (two for each cell in single cell data)
-    // Chain will be added in the header as cdr3Sequence.chain and used in python script
-    // Notice chain is in spec.domain for single cell data and spec.axesSpec[0].domain for bulk data
-
-	// Helper function to add chain information to the headers dynamically
-	chainMapping := {
-		"IG": { "A": "Heavy", "B": "Light" },
-		"TCRAB": { "A": "TRA", "B": "TRB" },
-		"TCRGD": { "A": "TRG", "B": "TRD" }
-	}
-
-	makeHeaderName := func(col, baseHeaderName, isSingleCell) {
-		if isSingleCell {
-			chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
-			receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
-			chainLabel := chainMapping[receptor][chain]
-			return baseHeaderName + "." + chainLabel // e.g., "cdr3Sequence.Heavy"
-		} else {
-			// For bulk, if chain info is available (e.g. IGH, IGK, IGL)
-			chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
-			if chainFromDomain != undefined {
-				return baseHeaderName + "." + chainFromDomain // e.g., "cdr3Sequence.IGH"
-			}
-		}
-		return baseHeaderName
-	};
-
-	// Process CDR3 sequences
-	cdr3Sequences := columns.getColumns("cdr3Sequences")
-
-	for col in cdr3Sequences {
-		headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
-		cdr3SeqTable.add(col, {header: headerName})
-	}
-
-	// Process V genes
-	vGenes := columns.getColumns("VGenes")	
-
-	for col in vGenes {
-		headerName := makeHeaderName(col, "vGene", isSingleCell)
-		cdr3SeqTable.add(col, {header: headerName})
-	}
-
-	// Process J genes
-	jGenes := columns.getColumns("JGenes")	
-
-	for col in jGenes {
-		headerName := makeHeaderName(col, "jGene", isSingleCell)
-		cdr3SeqTable.add(col, {header: headerName})
-	}
-
-	cdr3SeqTable.mem("16GiB")
-	cdr3SeqTable.cpu(1)
-	cdr3SeqTableBuilt := cdr3SeqTable.build()
-
-	cdr3VspectratypeCmd := exec.builder().
-		software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.spectratype:main")).
-		mem("16GiB").
-		cpu(1).
-		addFile("cdr3_sequences_input.tsv", cdr3SeqTableBuilt).
-		arg("--input_tsv").arg("cdr3_sequences_input.tsv").
-		arg("--spectratype_tsv").arg("spectratype.tsv").
-		arg("--vj_usage_tsv").arg("vj_usage.tsv") // no dot here
-
-	// Add top clonotypes argument and file to the builder if provided
-	if finalClonotypesCsv != undefined {
-		cdr3VspectratypeCmd = cdr3VspectratypeCmd.
-			arg("--final_clonotypes_csv").arg("finalClonotypes.csv").
-			addFile("finalClonotypes.csv", finalClonotypesCsv)
-	}
-
-	cdr3VspectratypeCmd = cdr3VspectratypeCmd. // continue building the command
-		saveFile("spectratype.tsv").
-		saveFile("vj_usage.tsv").
-		printErrStreamToStdout().
-		saveStdoutContent().
-		cache(24 * 60 * 60 * 1000).
-		run()
-
-
-	// Spectratype PFrame structure is [chain][cdr3Length][vGene] -> count
-
-	cdr3VspectratypePf := xsv.importFile(cdr3VspectratypeCmd.getFile("spectratype.tsv"), 
-										"tsv", spectratypeConv.getColumns(),
-										{cpu: 1, mem: "16GiB"})
-	outputs["cdr3VspectratypePf"] = pframes.exportFrame(cdr3VspectratypePf) 
-
-	// For vjUsage structure is [chain][vGene][jGene] -> count
-	vjUsagePf := xsv.importFile(cdr3VspectratypeCmd.getFile("vj_usage.tsv"), 
-								"tsv", vjUsageConv.getColumns(), 
-								{cpu: 1, mem: "16GiB"})
-	outputs["vjUsagePf"] = pframes.exportFrame(vjUsagePf)
-
-	return {
-		outputs: outputs,
-		exports: {}
-	}
-})

package/src/prerun.tpl.tengo

@@ -1,14 +1,24 @@
 // Prerun template for clonotype filtering
 wf := import("@platforma-sdk/workflow-tengo:workflow")
-render := import("@platforma-sdk/workflow-tengo:render")
-ll := import("@platforma-sdk/workflow-tengo:ll")
+exec := import("@platforma-sdk/workflow-tengo:exec")
 assets := import("@platforma-sdk/workflow-tengo:assets")
+xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
 pframes := import("@platforma-sdk/workflow-tengo:pframes")
 slices := import("@platforma-sdk/workflow-tengo:slices")
+render := import("@platforma-sdk/workflow-tengo:render")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+spectratypeConv := import(":pf-spectratype-conv")
+vjUsageConv := import(":pf-vj-usage-conv")
 
 filterAndSampleTpl := assets.importTemplate(":filter-and-sample")
 
 wf.prepare(func(args){
+	if is_undefined(args.inputAnchor) {
+        return {
+            columns: wf.createPBundleBuilder().build()
+        }
+    }
 	// We need a table with cluster ID (optional) | clonotype id | selected ranking columns
     bundleBuilder := wf.createPBundleBuilder()
     bundleBuilder.ignoreMissingDomains() // to make query work for both bulk and single cell data
@@ -98,8 +108,13 @@ wf.body(func(args) {
 	outputs := {}
 
     if !is_undefined(args.inputAnchor) {
+        // Input arguments
         columns := args.columns
         datasetSpec := columns.getSpec(args.inputAnchor)
+        topClonotypes := args.topClonotypes
+
+        // Needed conditional variable
+	    isSingleCell := datasetSpec.axesSpec[1].name == "pl7.app/vdj/scClonotypeKey"
     
         ////////// Clonotype Filtering //////////
         // Build clonotype table
@@ -217,17 +232,119 @@ wf.body(func(args) {
         filterSampleResult := render.create(filterAndSampleTpl, {
             inputAnchor: args.inputAnchor,
             cloneTable: cloneTable,
-            topClonotypes: args.topClonotypes,
             rankingOrder: args.rankingOrder,
             rankingOrderDefault: args.rankingOrderDefault,
             filters: args.filters,
             filterMap: filterMap,
             rankingMap: rankingMap,
-            datasetSpec: datasetSpec
+            datasetSpec: datasetSpec,
+            topClonotypes: args.topClonotypes
         })
         
         // Get the filtered clonotypes from the template result
         outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
+
+        // Get the filtered and sampled clonotypes P-frame and CSV from the template result
+        finalClonotypesCsv := filterSampleResult.output("finalClonotypesCsv", 24 * 60 * 60 * 1000)
+        // outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
+        
+        ////////// CDR3 Length Calculation //////////
+        
+        cdr3SeqTable := pframes.tsvFileBuilder()
+        cdr3SeqTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
+
+        // Must deal with multiple CDR3 sequences (two for each cell in single cell data)
+        // Chain will be added in the header as cdr3Sequence.chain and used in python script
+        // Notice chain is in spec.domain for single cell data and spec.axesSpec[0].domain for bulk data
+
+        // Helper function to add chain information to the headers dynamically
+        chainMapping := {
+            "IG": { "A": "Heavy", "B": "Light" },
+            "TCRAB": { "A": "TRA", "B": "TRB" },
+            "TCRGD": { "A": "TRG", "B": "TRD" }
+        }
+
+        makeHeaderName := func(col, baseHeaderName, isSingleCell) {
+            if isSingleCell {
+                chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
+                receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
+                chainLabel := chainMapping[receptor][chain]
+                return baseHeaderName + "." + chainLabel // e.g., "cdr3Sequence.Heavy"
+            } else {
+                // For bulk, if chain info is available (e.g. IGH, IGK, IGL)
+                chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
+                if chainFromDomain != undefined {
+                    return baseHeaderName + "." + chainFromDomain // e.g., "cdr3Sequence.IGH"
+                }
+            }
+            return baseHeaderName
+        };
+
+        // Process CDR3 sequences
+        cdr3Sequences := columns.getColumns("cdr3Sequences")
+
+        for col in cdr3Sequences {
+            headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
+            cdr3SeqTable.add(col, {header: headerName})
+        }
+
+        // Process V genes
+        vGenes := columns.getColumns("VGenes")	
+
+        for col in vGenes {
+            headerName := makeHeaderName(col, "vGene", isSingleCell)
+            cdr3SeqTable.add(col, {header: headerName})
+        }
+
+        // Process J genes
+        jGenes := columns.getColumns("JGenes")	
+
+        for col in jGenes {
+            headerName := makeHeaderName(col, "jGene", isSingleCell)
+            cdr3SeqTable.add(col, {header: headerName})
+        }
+
+        cdr3SeqTable.mem("16GiB")
+        cdr3SeqTable.cpu(1)
+        cdr3SeqTableBuilt := cdr3SeqTable.build()
+
+        cdr3VspectratypeCmd := exec.builder().
+            software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.spectratype:main")).
+            mem("16GiB").
+            cpu(1).
+            addFile("cdr3_sequences_input.tsv", cdr3SeqTableBuilt).
+            arg("--input_tsv").arg("cdr3_sequences_input.tsv").
+            arg("--spectratype_tsv").arg("spectratype.tsv").
+            arg("--vj_usage_tsv").arg("vj_usage.tsv") // no dot here
+
+        // Add top clonotypes argument and file to the builder if provided
+        if finalClonotypesCsv != undefined {
+            cdr3VspectratypeCmd = cdr3VspectratypeCmd.
+                arg("--final_clonotypes_csv").arg("finalClonotypes.csv").
+                addFile("finalClonotypes.csv", finalClonotypesCsv)
+        }
+
+        cdr3VspectratypeCmd = cdr3VspectratypeCmd. // continue building the command
+            saveFile("spectratype.tsv").
+            saveFile("vj_usage.tsv").
+            printErrStreamToStdout().
+            saveStdoutContent().
+            cache(24 * 60 * 60 * 1000).
+            run()
+
+
+        // Spectratype PFrame structure is [chain][cdr3Length][vGene] -> count
+
+        cdr3VspectratypePf := xsv.importFile(cdr3VspectratypeCmd.getFile("spectratype.tsv"), 
+                                            "tsv", spectratypeConv.getColumns(),
+                                            {cpu: 1, mem: "16GiB"})
+        outputs["cdr3VspectratypePf"] = pframes.exportFrame(cdr3VspectratypePf) 
+
+        // For vjUsage structure is [chain][vGene][jGene] -> count
+        vjUsagePf := xsv.importFile(cdr3VspectratypeCmd.getFile("vj_usage.tsv"), 
+                                    "tsv", vjUsageConv.getColumns(), 
+                                    {cpu: 1, mem: "16GiB"})
+        outputs["vjUsagePf"] = pframes.exportFrame(vjUsagePf)
     }
 
 	return {