@platforma-open/milaboratories.top-antibodies.workflow 1.1.1 → 1.2.0

package/.turbo/turbo-build.log

@@ -1,13 +1,17 @@
  WARN  Issue while reading "/home/runner/work/top-antibodies/top-antibodies/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.top-antibodies.workflow@1.1.1 build /home/runner/work/top-antibodies/top-antibodies/workflow
+> @platforma-open/milaboratories.top-antibodies.workflow@1.2.0 build /home/runner/work/top-antibodies/top-antibodies/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/main.tpl.tengo"...
+Processing "src/pf-umap-conv.lib.tengo"...
 Processing "src/sampled-cols-conv.lib.tengo"...
+Processing "src/sampled-cols-umap-conv.lib.tengo"...
 No syntax errors found.
   info: Compiling 'dist'...
+  info:   - writing /home/runner/work/top-antibodies/top-antibodies/workflow/dist/tengo/lib/pf-umap-conv.lib.tengo
   info:   - writing /home/runner/work/top-antibodies/top-antibodies/workflow/dist/tengo/lib/sampled-cols-conv.lib.tengo
+  info:   - writing /home/runner/work/top-antibodies/top-antibodies/workflow/dist/tengo/lib/sampled-cols-umap-conv.lib.tengo
   info:   - writing /home/runner/work/top-antibodies/top-antibodies/workflow/dist/tengo/tpl/main.plj.gz
   info: Template Pack build done.
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @platforma-open/milaboratories.top-antibodies.workflow
 
+## 1.2.0
+
+### Minor Changes
+
+- 5ee90ac: Add CDR3 spectratype
+
+### Patch Changes
+
+- Updated dependencies [5ee90ac]
+  - @platforma-open/milaboratories.top-antibodies.spectratype@1.1.0
+
 ## 1.1.1
 
 ### Patch Changes

package/dist/tengo/lib/pf-umap-conv.lib.tengo

@@ -0,0 +1,43 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+getColumns := func(datasetSpec) {
+  return {
+    "axes": [
+      {
+        "column": "clonotypeKey",
+        "spec": datasetSpec.axesSpec[1]
+      }
+      ],
+    "columns": [
+      {
+        "column": "UMAP1",
+        "id": "umap1",
+        "allowNA": false,
+        "spec": {
+          "name": "pl7.app/vdj/umap1",
+          "valueType": "Double",
+          "annotations": {
+            "pl7.app/label": "UMAP Dim1"
+          }
+        }
+      },
+      {
+        "column": "UMAP2",
+        "id": "umap2",
+        "allowNA": false,
+        "spec": {
+          "name": "pl7.app/vdj/umap2",
+          "valueType": "Double",
+          "annotations": {
+            "pl7.app/label": "UMAP Dim2"
+          }
+        }
+      }],
+    "storageFormat": "Binary",
+    "partitionKeyLength": 0
+  }
+}
+
+export ll.toStrict({
+	getColumns: getColumns
+})

package/dist/tengo/lib/sampled-cols-conv.lib.tengo

@@ -9,11 +9,11 @@ getColumns := func(datasetSpec, linkerAxisSpec) {
 
   if len(linkerAxisSpec) > 0 {
     for key, val in linkerAxisSpec {
-      axes.push([
-      {
-        "column": key,
-        "spec": val
-      }])
+      axes = axes + [
+        {
+          "column": key,
+          "spec": val
+        }]
     }
     
   }
@@ -25,12 +25,11 @@ getColumns := func(datasetSpec, linkerAxisSpec) {
         "id": "link",
         "allowNA": false,
         "spec": {
-          "name": "pl7.app/vdj/link",
+          "name": "pl7.app/vdj/sampling-column",
           "valueType": "Int",
           "domain": {},
           "annotations": {
-            "pl7.app/isLinkerColumn": "true",
-            "pl7.app/label": "Linker column",
+            "pl7.app/label": "Sampling column",
             "pl7.app/table/visibility": "optional"
           }
         }

package/dist/tengo/lib/sampled-cols-umap-conv.lib.tengo

@@ -0,0 +1,38 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+
+
+getColumns := func(datasetSpec) {
+  axes := [
+      {
+        "column": "clonotypeKey",
+        "spec": datasetSpec.axesSpec[1]
+      }]
+
+  return {
+    "axes": axes,
+    "columns": [
+      {
+        "column": "top",
+        "id": "link2", // temporal change
+        "allowNA": false,
+        "spec": {
+          "name": "pl7.app/vdj/sampling-column-umap",
+          "valueType": "Int",
+          "domain": {},
+          "annotations": {
+            "pl7.app/label": "Top clonotypes",
+            "pl7.app/table/visibility": "optional",
+            "pl7.app/isSubset": "true"
+          }
+        }
+      }
+    ],
+    "storageFormat": "Binary",
+    "partitionKeyLength": 0
+  }
+}
+
+export ll.toStrict({
+	getColumns: getColumns
+})

package/package.json

@@ -1,15 +1,17 @@
 {
   "name": "@platforma-open/milaboratories.top-antibodies.workflow",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
     "@platforma-sdk/workflow-tengo": "^4.3.2",
-    "@platforma-open/milaboratories.top-antibodies.software": "1.0.1"
+    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.0.1",
+    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.1.0",
+    "@platforma-open/milaboratories.top-antibodies.umap": "1.0.1"
   },
   "devDependencies": {
     "@platforma-sdk/tengo-builder": "^2.1.3",
-    "@platforma-sdk/test": "^1.30.19",
+    "@platforma-sdk/test": "^1.30.24",
     "vitest": "^2.1.8"
   },
   "scripts": {

package/src/main.tpl.tengo

@@ -5,7 +5,9 @@ assets:= import("@platforma-sdk/workflow-tengo:assets")
 xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
 pframes := import("@platforma-sdk/workflow-tengo:pframes")
 sampledColsConv := import(":sampled-cols-conv")
-
+sampledColsUmapConv := import(":sampled-cols-umap-conv")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+umapConv := import(":pf-umap-conv")
 
 wf.prepare(func(args){
 	// We need a table with cluster ID (optional) | clonotype id | selected ranking columns
@@ -18,12 +20,43 @@ wf.prepare(func(args){
 		bundleBuilder.addSingle(col)
 	}
 
+	// Add linker column
+	bundleBuilder.addMulti({
+		axes: [{ anchor: "main", idx: 1 }], // this will do partial axes match (unlike in the model)
+		annotations: { "pl7.app/isLinkerColumn": "true" },
+		partialAxesMatch: true
+	}, "linkers")
+
+	// Add full aa sequence column
 	bundleBuilder.addMulti({
 		axes: [{ anchor: "main", idx: 1 }],
-		partialAxesMatch: true,
-		annotations: { "pl7.app/isLinkerColumn": "true" }
+		annotations: {
+			"pl7.app/vdj/isAssemblingFeature": "true",
+			"pl7.app/vdj/isMainSequence": "true"
 		},
-		"linker")
+		domain: {
+			"pl7.app/alphabet": "aminoacid"
+		}
+	}, "aaSequence")
+
+	// Add CDR3 sequences
+	bundleBuilder.addMulti({
+		axes: [{ anchor: "main", idx: 1 }], // Clonotype axis
+		name: "pl7.app/vdj/sequence",
+		domain: {
+			"pl7.app/alphabet": "aminoacid",
+			"pl7.app/vdj/feature": "CDR3"    // Specify CDR3 feature
+		}
+	}, "cdr3Sequences") // New collection name for CDR3 sequences
+
+	// Add V gene
+	bundleBuilder.addMulti({
+		axes: [{ anchor: "main", idx: 1 }], // Clonotype axis
+		name: "pl7.app/vdj/geneHit",
+		domain: {
+			"pl7.app/vdj/reference": "VGene"
+		}
+	}, "VGenes")
 
 	return {
 		columns: bundleBuilder.build()
@@ -32,57 +65,245 @@ wf.prepare(func(args){
 
 wf.body(func(args) {
 
+	// Input arguments
 	columns := args.columns
 	datasetSpec := columns.getSpec(args.inputAnchor)
+	
 	topClonotypes := args.topClonotypes
+
+	// Needed conditional variables
+	isSingleCell := datasetSpec.axesSpec[1].name == "pl7.app/vdj/scClonotypeKey"
 	
+	// output containers 
+	outputs := {}
 
-	clonoTable := columns.xsvTableBuilder()
-	clonoTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
+	////////// Clonotype Filtering //////////
+	
+	// Build clonotype table
+	cloneTable := columns.xsvTableBuilder()
+	cloneTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
 	
 	for i, col in args.rankingOrder {
-		clonoTable.add(col, {header: "Col" + string(i)})
+		cloneTable.add(col, {header: "Col" + string(i)})
+	
+		// If column does not have main anchor axis we have to include theirs
+		colsSpec := columns.getSpec(col)
+		axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+		if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+			for na, ax in colsSpec.axesSpec {
+				if ax.name != datasetSpec.axesSpec[1].name {
+					cloneTable.setAxisHeader(ax.name, "cluster_" + string(i) + string(na))
+				}
+			}
+		}	
 	}
 
 	// Columns gotten by query require .key
 	linkerAxisSpec := {}
-	for i, col in columns.getColumns("linker") {
+	for i, col in columns.getColumns("linkers") {
 		if datasetSpec.axesSpec[1].name == col.spec.axesSpec[1].name {
-			clonoTable.add(col.key, {header: "linker." + string(i)})
-			clonoTable.setAxisHeader(col.spec.axesSpec[0].name, "cluster_" + string(i))
+			cloneTable.add(col.key, {header: "linker." + string(i)})
+			cloneTable.setAxisHeader(col.spec.axesSpec[0].name, "cluster_" + string(i))
 			linkerAxisSpec["cluster_" + string(i)] = col.spec.axesSpec[0]
 		} else if datasetSpec.axesSpec[1].name == col.spec.axesSpec[0].name {
-			clonoTable.add(col.key, {header: "linker." + string(i)})
-			clonoTable.setAxisHeader(col.spec.axesSpec[1].name, "cluster_" + string(i))
+			cloneTable.add(col.key, {header: "linker." + string(i)})
+			cloneTable.setAxisHeader(col.spec.axesSpec[1].name, "cluster_" + string(i))
 			linkerAxisSpec["cluster_" + string(i)] = col.spec.axesSpec[1]
 		}
-		
 	}
+	cloneTable = cloneTable.build("csv")
 
-	clonoTable = clonoTable.build("csv")
+	if topClonotypes != undefined {
+		// Run sampling script
+		sampleClones := exec.builder().
+			software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.sample-clonotypes:main")).
+			addFile("filteredClonotypes.csv", cloneTable).
+			arg("--csv").arg("filteredClonotypes.csv").
+			arg("--n").arg(string(topClonotypes)).
+			arg("--out").arg("sampledClonotypes.csv").
+			saveFile("sampledClonotypes_top.csv").
+			printErrStreamToStdout().
+			saveStdoutContent().
+			cache(24 * 60 * 60 * 1000).
+			run()
+		
+		// Store outputs
+		sampledColsParams := sampledColsConv.getColumns(datasetSpec, linkerAxisSpec)
+		sampledColumnsPf := xsv.importFile(sampleClones.getFile("sampledClonotypes_top.csv"), "csv", sampledColsParams)
+		outputs["sampledRows"] = pframes.exportFrame(sampledColumnsPf)
 
-	sampleClones := exec.builder().
-		software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.software:sample-clonotypes")).
-		addFile("filteredClonotypes.csv", clonoTable).
-		arg("--csv").arg("filteredClonotypes.csv").
-		arg("--n").arg(string(topClonotypes)).
-		arg("--out").arg("sampledClonotypes.csv").
-		saveFile("sampledClonotypes_top.csv").
+    	// Prepare filter col subset for UMAP
+    	// Avoid taking cluster axis using other params
+    	sampledColsUmapParams := sampledColsUmapConv.getColumns(datasetSpec) 
+    	sampledColsUmapPf := xsv.importFile(sampleClones.getFile("sampledClonotypes_top.csv"), "csv", sampledColsUmapParams)
+		outputs["sampledRowsUmap"] = pframes.exportFrame(sampledColsUmapPf)
+	}
+	
+	////////// UMAP //////////
+	// Generate input TSV with Clonotype ID and aa sequence
+	umapTable := columns.xsvTableBuilder()
+	umapTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
+	for col in columns.getColumns("aaSequence") {
+		if isSingleCell {
+			chainLabel := col.spec.domain["pl7.app/vdj/scClonotypeChain"]
+			umapTable.add(col.key, {header: "aaSequence." + chainLabel})
+		} else {
+			chainLabel := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"]
+			umapTable.add(col.key, {header: "aaSequence." + chainLabel})
+		}
+	}
+	umapTable = umapTable.build("tsv")
+
+	// UMAP script should go here
+	umapClones := exec.builder().
+		software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.umap:main")).
+		addFile("sequences.tsv", umapTable).
+		arg("-i").arg("sequences.tsv").
+		arg("-u").arg("umap.tsv").
+		saveFile("umap.tsv").
 		printErrStreamToStdout().
 		saveStdoutContent().
 		cache(24 * 60 * 60 * 1000).
 		run()
+
+
+	umapPf := xsv.importFile(umapClones.getFile("umap.tsv"), "tsv", umapConv.getColumns(datasetSpec))
+	outputs["umap"] = pframes.exportFrame(umapPf)
+
+	////////// CDR3 Length Calculation //////////
 	
-	sampledColsParams := sampledColsConv.getColumns(datasetSpec, linkerAxisSpec)
-	sampledColumnsPf := xsv.importFile(sampleClones.getFile("sampledClonotypes_top.csv"), "csv", sampledColsParams)
+	cdr3SeqTable := columns.xsvTableBuilder()
+	cdr3SeqTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
 
-	// we need to output all filtered data axis
+    // Must deal with multiple CDR3 sequences (two for each cell in single cell data)
+    // Chain will be added in the header as cdr3Sequence.chain and used in python script
+    // Notice chain is in spec.domain for single cell data and spec.axesSpec[0].domain for bulk data
 
-	return {
-		outputs: {
-			"sampledColumns": pframes.exportFrame(sampledColumnsPf)
+	// Helper function to add chain information to the headers dynamically
+	chainMapping := {
+		"IG": { "A": "Heavy", "B": "Light" },
+		"TCRAB": { "A": "TRA", "B": "TRB" },
+		"TCRGD": { "A": "TRG", "B": "TRD" }
+	}
+
+	makeHeaderName := func(col, baseHeaderName, isSingleCell) {
+		if isSingleCell {
+			chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
+			receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
+			chainLabel := chainMapping[receptor][chain]
+			return baseHeaderName + "." + chainLabel
+		} else {
+			// For bulk, if chain info is available (e.g. IGH, IGK, IGL)
+			chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
+			if chainFromDomain != undefined {
+				return baseHeaderName + "." + chainFromDomain
+			}
+		}
+		return baseHeaderName // Default header for bulk
+	};
+
+	// Process CDR3 sequences
+	cdr3Sequences := columns.getColumns("cdr3Sequences")
+
+	for col in cdr3Sequences {
+		headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
+		cdr3SeqTable.add(col.key, {header: headerName})
+	}
+
+	// Process V genes
+	vGenes := columns.getColumns("VGenes")	
+
+	for col in vGenes {
+		headerName := makeHeaderName(col, "vGene", isSingleCell)
+		cdr3SeqTable.add(col.key, {header: headerName})
+	}
+
+	cdr3SeqTableBuilt := cdr3SeqTable.build("tsv")
+
+	cdr3VspectratypeCmd := exec.builder().
+		software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.spectratype:main")).
+		addFile("cdr3_sequences_input.tsv", cdr3SeqTableBuilt).
+		arg("--input_tsv").arg("cdr3_sequences_input.tsv").
+		arg("--output_tsv").arg("cdr3_lengths.tsv").
+		saveFile("cdr3_lengths.tsv").
+		printErrStreamToStdout().
+		saveStdoutContent().
+		cache(24 * 60 * 60 * 1000).
+		run()
+
+
+	// For spectratype structure is:
+	// [chain][cdr3Length][vGene] -> count
+
+	// Get the spec for the vGene column
+	vGeneSpec := columns.getSpec(vGenes[0].key)
+
+	// Set up the axes for the output data
+	axes := [
+		{
+			column: "chain",
+			spec: {
+				name: "pl7.app/vdj/chain",
+				type: "String", // For axis it is type, not valueType 
+				annotations: { "pl7.app/label": "CDR3 chain" }
+			}
+		},
+				{
+			column: "cdr3Length",
+			spec: {
+				name: "pl7.app/vdj/sequenceLength",
+				type: "Int",
+				domain: {
+					"pl7.app/vdj/feature": "CDR3",
+					"pl7.app/alphabet": "aminoacid"
+				},
+				annotations: { "pl7.app/label": "CDR3 aa Length" }
+			}
 		},
+		{
+			column: "vGene",
+			spec: {
+				name: "pl7.app/vdj/geneHit",
+				type: "String",
+				domain: vGeneSpec.domain,
+				annotations: {
+					"pl7.app/label": "Best V gene"
+				}
+			}
+		}
+		
+	]
+
+
+	spectratypeColumns := [
+		{
+			column: "count",
+			spec: {
+				name: "pl7.app/vdj/vSpectratype",
+				valueType: "Int",
+				domain: {
+					"pl7.app/vdj/feature": "CDR3",
+					"pl7.app/alphabet": "aminoacid"
+				},
+				annotations: { "pl7.app/label": "CDR3 V Spectratype" }
+			}
+		}
+	]
+
+	spectratypeSpec := {
+		axes: axes,
+		columns: spectratypeColumns,
+		storageFormat: "Binary",
+		partitionKeyLength: 0
+	}
+
+	cdr3VspectratypePf := xsv.importFile(cdr3VspectratypeCmd.getFile("cdr3_lengths.tsv"), "tsv", spectratypeSpec)
+	
+	outputs["cdr3VspectratypePf"] = pframes.exportFrame(cdr3VspectratypePf) 
+	//ll.print("CDR3 lengths PFrame imported.")
+
+	return {
+		outputs: outputs,
 		exports: {}
 	}
 })
-

package/src/pf-umap-conv.lib.tengo

@@ -0,0 +1,43 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+getColumns := func(datasetSpec) {
+  return {
+    "axes": [
+      {
+        "column": "clonotypeKey",
+        "spec": datasetSpec.axesSpec[1]
+      }
+      ],
+    "columns": [
+      {
+        "column": "UMAP1",
+        "id": "umap1",
+        "allowNA": false,
+        "spec": {
+          "name": "pl7.app/vdj/umap1",
+          "valueType": "Double",
+          "annotations": {
+            "pl7.app/label": "UMAP Dim1"
+          }
+        }
+      },
+      {
+        "column": "UMAP2",
+        "id": "umap2",
+        "allowNA": false,
+        "spec": {
+          "name": "pl7.app/vdj/umap2",
+          "valueType": "Double",
+          "annotations": {
+            "pl7.app/label": "UMAP Dim2"
+          }
+        }
+      }],
+    "storageFormat": "Binary",
+    "partitionKeyLength": 0
+  }
+}
+
+export ll.toStrict({
+	getColumns: getColumns
+})

package/src/sampled-cols-conv.lib.tengo

@@ -9,11 +9,11 @@ getColumns := func(datasetSpec, linkerAxisSpec) {
 
   if len(linkerAxisSpec) > 0 {
     for key, val in linkerAxisSpec {
-      axes.push([
-      {
-        "column": key,
-        "spec": val
-      }])
+      axes = axes + [
+        {
+          "column": key,
+          "spec": val
+        }]
     }
     
   }
@@ -25,12 +25,11 @@ getColumns := func(datasetSpec, linkerAxisSpec) {
         "id": "link",
         "allowNA": false,
         "spec": {
-          "name": "pl7.app/vdj/link",
+          "name": "pl7.app/vdj/sampling-column",
           "valueType": "Int",
           "domain": {},
           "annotations": {
-            "pl7.app/isLinkerColumn": "true",
-            "pl7.app/label": "Linker column",
+            "pl7.app/label": "Sampling column",
             "pl7.app/table/visibility": "optional"
           }
         }

package/src/sampled-cols-umap-conv.lib.tengo

@@ -0,0 +1,38 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+// modified from sampled-cols-conv.lib.tengo
+
+getColumns := func(datasetSpec) {
+  axes := [
+      {
+        "column": "clonotypeKey",
+        "spec": datasetSpec.axesSpec[1]
+      }]
+
+  return {
+    "axes": axes,
+    "columns": [
+      {
+        "column": "top",
+        "id": "link2", // temporal change
+        "allowNA": false,
+        "spec": {
+          "name": "pl7.app/vdj/sampling-column-umap",
+          "valueType": "Int",
+          "domain": {},
+          "annotations": {
+            "pl7.app/label": "Top clonotypes",
+            "pl7.app/table/visibility": "optional",
+            "pl7.app/isSubset": "true"
+          }
+        }
+      }
+    ],
+    "storageFormat": "Binary",
+    "partitionKeyLength": 0
+  }
+}
+
+export ll.toStrict({
+	getColumns: getColumns
+})