@platforma-open/milaboratories.top-antibodies.workflow 1.13.2 → 1.14.0

package/src/libs/data-utils.lib.tengo

@@ -0,0 +1,324 @@
+// Data utility functions for clonotype filtering and processing
+slices := import("@platforma-sdk/workflow-tengo:slices")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+times := import("times")
+text := import("text")
+
+// Helper function to add chain information to the headers dynamically
+makeHeaderName := func(col, baseHeaderName, isSingleCell) {
+    chainMapping := {
+        "IG": { "A": "Heavy", "B": "Light" },
+        "TCRAB": { "A": "TRA", "B": "TRB" },
+        "TCRGD": { "A": "TRG", "B": "TRD" }
+    }
+    
+    if isSingleCell {
+        chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
+        receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
+        chainLabel := chainMapping[receptor][chain]
+        return baseHeaderName + "." + chainLabel // e.g., "cdr3Sequence.Heavy"
+    } else {
+        // For bulk, if chain info is available (e.g. IGH, IGK, IGL)
+        chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
+        if chainFromDomain != undefined {
+            return baseHeaderName + "." + chainFromDomain // e.g., "cdr3Sequence.IGH"
+        }
+    }
+    return baseHeaderName
+}
+
+// Prepare clonotype data: filters, ranking columns, linkers, cluster sizes
+prepareClonotypeData := func(filters, rankingOrder, rankingOrderDefault, columns, datasetSpec) {
+    structuredMap := {}
+    axisRenames := {}
+    filterMap := {}
+    rankingMap := {}
+    addedAxes := []
+    addedCols := false
+    linkerAxisSpec := {}
+    
+    // Add Filters to table
+    if len(filters) > 0 {
+        for i, filter in filters {
+            if filter.value != undefined {
+                // Columns added here might also be in ranking list, so we add default IDs
+                col := columns.getColumn(filter.value.column)
+                structuredMap["Filter_" + string(i)] = { spec: col.spec, data: col.data }            
+                addedCols = true
+                // Store reference value and filter type associated to this column
+                filterMap["Filter_" + string(i)] = filter.filter
+                filterMap["Filter_" + string(i)]["valueType"] = columns.getSpec(filter.value.column).valueType
+                
+                // If column does not have main anchor axis we have to include theirs
+                colsSpec := columns.getSpec(filter.value.column)
+                axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+                if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+                    for na, ax in colsSpec.axesSpec {
+                        if ax.name != datasetSpec.axesSpec[1].name {
+                            axisAlias := "cluster_" + string(i) + string(na)
+                            axisRenames[ax.name] = axisAlias
+                            addedAxes = append(addedAxes, ax.name)
+                        }
+                    }
+                }	
+            }
+        }
+    }
+
+    // Add ranking columns to table
+    validRanks := false
+    if len(rankingOrder) > 0 {
+        for i, rankCol in rankingOrder {
+            if rankCol.value != undefined {
+                validRanks = true
+                col := columns.getColumn(rankCol.value.column)
+                structuredMap["Col" + string(i)] = { spec: col.spec, data: col.data }
+                addedCols = true
+                // Store ranking order for this column
+                rankingMap["Col" + string(i)] = rankCol.rankingOrder
+                
+                // If column does not have main anchor axis we have to include theirs
+                colsSpec := columns.getSpec(rankCol.value.column)
+                axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+                if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+                    for na, ax in colsSpec.axesSpec {
+                        if ax.name != datasetSpec.axesSpec[1].name && !slices.hasElement(addedAxes, ax.name) {
+                            axisAlias := "cluster_" + string(i) + string(na)
+                            axisRenames[ax.name] = axisAlias
+                        }
+                    }
+                }	
+            }
+        }
+    }
+    // If we didn't have any ranking column or all where not valid
+    if !validRanks {
+        // @TODO: this is a temporal patch for issue where rankingOrderDefault 
+        // are not defined by the time prerun works
+        if rankingOrderDefault.value != undefined {
+            i := 0
+            col := columns.getColumn(rankingOrderDefault.value.column)
+            structuredMap["Col" + string(i)] = { spec: col.spec, data: col.data }
+            addedCols = true
+            // Store default ranking order
+            rankingMap["Col" + string(i)] = rankingOrderDefault.rankingOrder
+        
+            // If column does not have main anchor axis we have to include theirs
+            colsSpec := columns.getSpec(rankingOrderDefault.value.column)
+            axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+            if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+                for na, ax in colsSpec.axesSpec {
+                    if ax.name != datasetSpec.axesSpec[1].name {
+                        axisAlias := "cluster_" + string(i) + string(na)
+                        axisRenames[ax.name] = axisAlias
+                    }
+                }
+            }	
+        }
+    }
+
+    // Get linker columns if needed
+    if len(columns.getColumns("linkers")) > 0 {
+        for i, col in columns.getColumns("linkers") {
+            if datasetSpec.axesSpec[1].name == col.spec.axesSpec[1].name {
+                structuredMap["linker." + string(i)] = { spec: col.spec, data: col.data }
+                axisAlias := "cluster_" + string(i)
+                axisRenames[col.spec.axesSpec[0].name] = axisAlias
+                linkerAxisSpec[axisAlias] = col.spec.axesSpec[0]
+            } else if datasetSpec.axesSpec[1].name == col.spec.axesSpec[0].name {
+                structuredMap["linker." + string(i)] = { spec: col.spec, data: col.data }
+                axisAlias := "cluster_" + string(i)
+                axisRenames[col.spec.axesSpec[1].name] = axisAlias
+                linkerAxisSpec[axisAlias] = col.spec.axesSpec[1]
+            }
+            addedCols = true
+        }
+    }
+
+    // Add cluster size columns if available
+    if len(columns.getColumns("clusterSizes")) > 0 {
+        for i, col in columns.getColumns("clusterSizes") {
+            structuredMap["clusterSize." + string(i)] = { spec: col.spec, data: col.data }
+            addedCols = true
+            // Add the cluster axis header
+            for axisIdx, axis in col.spec.axesSpec {
+                if axis.name != datasetSpec.axesSpec[1].name {
+                    axisAlias := "clusterAxis_" + string(i) + "_" + string(axisIdx)
+                    axisRenames[axis.name] = axisAlias
+                }
+            }
+        }
+    }
+    
+    return {
+        structuredMap: structuredMap,
+        axisRenames: axisRenames,
+        filterMap: filterMap,
+        rankingMap: rankingMap,
+        addedCols: addedCols,
+        linkerAxisSpec: linkerAxisSpec
+    }
+}
+
+// Prepare CDR3 sequence data: CDR3 sequences, V genes, J genes
+prepareCdr3Data := func(columns, datasetSpec, isSingleCell) {
+    cdr3SeqStructuredMap := {}
+    cdr3SeqAxisRenames := {}
+    cdr3SeqAxisRenames[datasetSpec.axesSpec[1].name] = "clonotypeKey"
+
+    // Process CDR3 sequences
+    cdr3Sequences := columns.getColumns("cdr3Sequences")
+
+    for col in cdr3Sequences {
+        headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
+        if isSingleCell {
+            if col.spec.domain["pl7.app/vdj/scClonotypeChain/index"] == "primary" {
+                cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+            }
+        } else {
+            cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+        }
+    }
+
+    // Process V genes
+    vGenes := columns.getColumns("VGenes")	
+
+    for col in vGenes {
+        headerName := makeHeaderName(col, "vGene", isSingleCell)
+        cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+    }
+
+    // Process J genes
+    jGenes := columns.getColumns("JGenes")	
+
+    for col in jGenes {
+        headerName := makeHeaderName(col, "jGene", isSingleCell)
+        cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+    }
+    
+    return {
+        structuredMap: cdr3SeqStructuredMap,
+        axisRenames: cdr3SeqAxisRenames
+    }
+}
+
+// Ptabler-based filtering implementation
+// Replicates the logic from filter.py
+filterClonotypes := func(structuredMap, axisRenames, filterMap, datasetSpec) {
+    // Helper function to build filter predicate from filter spec
+    buildFilterPredicate := func(columnName, filterSpec) {
+        filterType := filterSpec["type"]
+        referenceValue := filterSpec["reference"]
+        col := pt.col(columnName)
+        
+        if filterType == "number_greaterThan" {
+            return col.gt(referenceValue)
+        } else if filterType == "number_greaterThanOrEqualTo" {
+            return col.ge(referenceValue)
+        } else if filterType == "number_lessThan" {
+            return col.lt(referenceValue)
+        } else if filterType == "number_lessThanOrEqualTo" {
+            return col.le(referenceValue)
+        } else if filterType == "number_equals" {
+            return col.eq(referenceValue)
+        } else if filterType == "number_notEquals" {
+            return col.neq(referenceValue)
+        } else if filterType == "string_equals" {
+            return col.eq(string(referenceValue))
+        } else if filterType == "string_notEquals" {
+            return col.neq(string(referenceValue))
+        } else if filterType == "string_contains" {
+            return col.strContains(string(referenceValue), {literal: true})
+        } else if filterType == "string_doesNotContain" {
+            return col.strContains(string(referenceValue), {literal: true}).not()
+        }
+        ll.panic("Unknown filter type: %s", filterType)
+        return undefined
+    }
+
+    // Build projection with axis renames and column selections
+    projection := []
+    // Add main clonotypeKey axis first
+    projection = append(projection, pt.axis(datasetSpec.axesSpec[1].name).alias("clonotypeKey"))
+    // Add other renamed axes
+    for origAxis, aliasName in axisRenames {
+        projection = append(projection, pt.axis(origAxis).alias(aliasName))
+    }
+    // Add all columns
+    for colName, _ in structuredMap {
+        projection = append(projection, pt.col(colName))
+    }
+
+    // Start ptabler workflow
+    wfPt := pt.workflow().cacheInputs(24 * 60 * 60 * 1000)
+    
+    // Create initial frame with axis renames
+    dfPt := wfPt.frame(pt.p.full(structuredMap)).select(projection...)
+
+    // Step 1: Apply filters from filterMap (matches filter.py lines 123-164)
+    // Only apply filters if the data type matches the filter type
+    filterPredicates := []
+    for filterColName, filterSpec in filterMap {
+        filterType := filterSpec["type"]
+        valueType := filterSpec["valueType"]
+        
+        // Validate filter type matches data type (same logic as filter.py)
+        isValidFilter := false
+        if valueType == "String" && text.has_prefix(filterType, "string_") {
+            isValidFilter = true
+        } else if valueType != "String" && text.has_prefix(filterType, "number_") {
+            isValidFilter = true
+        }
+        
+        if isValidFilter {
+            predicate := buildFilterPredicate(filterColName, filterSpec)
+            filterPredicates = append(filterPredicates, predicate)
+        }
+    }
+    
+    if len(filterPredicates) > 0 {
+        dfPt = dfPt.filter(filterPredicates...)
+    }
+
+    // Step 2: Add "top" column with value 1 (matches filter.py line 211)
+    dfPt = dfPt.withColumns(pt.lit(1).alias("top"))
+
+    // Save both CSV and pframe
+    frameParams := {
+        axes: [{
+            column: "clonotypeKey",
+            spec: datasetSpec.axesSpec[1]
+        }],
+        columns: [{
+            column: "top",
+            spec: {
+                name: "pl7.app/vdj/sampling-column",
+                valueType: "Int",
+                domain: {},
+                annotations: {
+                    "pl7.app/label": "Sampling column",
+                    "pl7.app/table/visibility": "optional",
+                    "pl7.app/isSubset": "true"
+                }
+            }
+        }]
+    }
+    
+    dfPt.save("filteredClonotypes.parquet")
+    dfPt.saveFrameDirect("filteredClonotypes", frameParams)
+    wfPtResult := wfPt.run()
+    
+    return {
+        filteredParquet: wfPtResult.getFile("filteredClonotypes.parquet"),
+        pframe: wfPtResult.getFrameDirect("filteredClonotypes")
+    }
+}
+
+export ll.toStrict({
+    makeHeaderName: makeHeaderName,
+    prepareClonotypeData: prepareClonotypeData,
+    prepareCdr3Data: prepareCdr3Data,
+    filterClonotypes: filterClonotypes
+})
+

package/src/{pf-kabat-conv.lib.tengo → libs/pf-kabat-conv.lib.tengo}

@@ -19,7 +19,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
         annotations: {
           "pl7.app/label": "KABAT sequence " + featureName + " Heavy",
           "pl7.app/table/orderPriority": "10",
-		      "pl7.app/table/visibility": "default"
+	      "pl7.app/table/visibility": "default"
         }
       }
     },
@@ -34,7 +34,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
         annotations: {
           "pl7.app/label": "KABAT positions " + featureName + " Heavy",
           "pl7.app/table/orderPriority": "9",
-		      "pl7.app/table/visibility": "optional"
+	      "pl7.app/table/visibility": "optional"
         }
       }
     }
@@ -53,7 +53,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
           annotations: {
             "pl7.app/label": "KABAT sequence " + featureName + " Light",
             "pl7.app/table/orderPriority": "8",
-		        "pl7.app/table/visibility": "default"
+	        "pl7.app/table/visibility": "default"
           }
         }
       },
@@ -68,7 +68,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
           annotations: {
             "pl7.app/label": "KABAT positions " + featureName + " Light",
             "pl7.app/table/orderPriority": "7",
-		        "pl7.app/table/visibility": "optional"
+	        "pl7.app/table/visibility": "optional"
           }
         }
       }
@@ -128,4 +128,3 @@ export ll.toStrict({
   getColumns: getColumns
 })
 
-

package/src/{pf-spectratype-conv.lib.tengo → libs/pf-spectratype-conv.lib.tengo}

@@ -57,3 +57,4 @@ getColumns := func() {
 export ll.toStrict({
     getColumns: getColumns
 })
+

package/{dist/tengo/lib → src/libs}/pf-vj-usage-conv.lib.tengo

@@ -52,3 +52,4 @@ getColumns := func() {
 export ll.toStrict({
     getColumns: getColumns
 })
+

package/src/{sampled-cols-conv.lib.tengo → libs/sampled-cols-conv.lib.tengo}

@@ -48,3 +48,4 @@ getColumns := func(datasetSpec, addRanking) {
 export ll.toStrict({
 	getColumns: getColumns
 })
+