@platforma-open/milaboratories.top-antibodies.workflow 1.13.1 → 1.14.0

package/.turbo/turbo-build.log

@@ -1,24 +1,24 @@
  WARN  Issue while reading "/home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.top-antibodies.workflow@1.13.1 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
+> @platforma-open/milaboratories.top-antibodies.workflow@1.14.0 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/assembling-fasta.tpl.tengo"...
-Processing "src/filter-and-sample.tpl.tengo"...
+Processing "src/libs/data-utils.lib.tengo"...
+Processing "src/libs/pf-kabat-conv.lib.tengo"...
+Processing "src/libs/pf-spectratype-conv.lib.tengo"...
+Processing "src/libs/pf-vj-usage-conv.lib.tengo"...
+Processing "src/libs/sampled-cols-conv.lib.tengo"...
 Processing "src/main.tpl.tengo"...
-Processing "src/pf-kabat-conv.lib.tengo"...
-Processing "src/pf-spectratype-conv.lib.tengo"...
-Processing "src/pf-vj-usage-conv.lib.tengo"...
 Processing "src/prerun.tpl.tengo"...
-Processing "src/sampled-cols-conv.lib.tengo"...
 No syntax errors found.
   info: Compiling 'dist'...
-  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/pf-kabat-conv.lib.tengo
-  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/pf-spectratype-conv.lib.tengo
-  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/pf-vj-usage-conv.lib.tengo
-  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/sampled-cols-conv.lib.tengo
+  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/libs.data-utils.lib.tengo
+  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/libs.pf-kabat-conv.lib.tengo
+  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/libs.pf-spectratype-conv.lib.tengo
+  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/libs.pf-vj-usage-conv.lib.tengo
+  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/libs.sampled-cols-conv.lib.tengo
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/assembling-fasta.plj.gz
-  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/filter-and-sample.plj.gz
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/prerun.plj.gz
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/main.plj.gz
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @platforma-open/milaboratories.top-antibodies.workflow
 
+## 1.14.0
+
+### Minor Changes
+
+- 532b9ed: Block performance optimization
+
+### Patch Changes
+
+- Updated dependencies [532b9ed]
+  - @platforma-open/milaboratories.top-antibodies.sample-clonotypes@1.7.0
+  - @platforma-open/milaboratories.top-antibodies.assembling-fasta@1.2.0
+  - @platforma-open/milaboratories.top-antibodies.anarci-kabat@1.2.0
+  - @platforma-open/milaboratories.top-antibodies.spectratype@1.7.0
+  - @platforma-open/milaboratories.top-antibodies.umap@1.2.0
+
+## 1.13.2
+
+### Patch Changes
+
+- e17b19a: Remove unused `saveStdoutContent` calls, update sdk
+
 ## 1.13.1
 
 ### Patch Changes

package/dist/index.cjs

@@ -1,6 +1,5 @@
 module.exports = { Templates: {
   'assembling-fasta': { type: 'from-file', path: require.resolve('./tengo/tpl/assembling-fasta.plj.gz') },
-  'filter-and-sample': { type: 'from-file', path: require.resolve('./tengo/tpl/filter-and-sample.plj.gz') },
   'prerun': { type: 'from-file', path: require.resolve('./tengo/tpl/prerun.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "assembling-fasta" | "filter-and-sample" | "prerun" | "main";
+declare type TplName = "assembling-fasta" | "prerun" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,7 +1,6 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'assembling-fasta': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/assembling-fasta.plj.gz') },
-  'filter-and-sample': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/filter-and-sample.plj.gz') },
   'prerun': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/prerun.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/dist/tengo/lib/libs.data-utils.lib.tengo

@@ -0,0 +1,324 @@
+
+slices := import("@platforma-sdk/workflow-tengo:slices")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+times := import("times")
+text := import("text")
+
+
+makeHeaderName := func(col, baseHeaderName, isSingleCell) {
+    chainMapping := {
+        "IG": { "A": "Heavy", "B": "Light" },
+        "TCRAB": { "A": "TRA", "B": "TRB" },
+        "TCRGD": { "A": "TRG", "B": "TRD" }
+    }
+    
+    if isSingleCell {
+        chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
+        receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
+        chainLabel := chainMapping[receptor][chain]
+        return baseHeaderName + "." + chainLabel // e.g., "cdr3Sequence.Heavy"
+    } else {
+
+        chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
+        if chainFromDomain != undefined {
+            return baseHeaderName + "." + chainFromDomain // e.g., "cdr3Sequence.IGH"
+        }
+    }
+    return baseHeaderName
+}
+
+
+prepareClonotypeData := func(filters, rankingOrder, rankingOrderDefault, columns, datasetSpec) {
+    structuredMap := {}
+    axisRenames := {}
+    filterMap := {}
+    rankingMap := {}
+    addedAxes := []
+    addedCols := false
+    linkerAxisSpec := {}
+    
+
+    if len(filters) > 0 {
+        for i, filter in filters {
+            if filter.value != undefined {
+
+                col := columns.getColumn(filter.value.column)
+                structuredMap["Filter_" + string(i)] = { spec: col.spec, data: col.data }            
+                addedCols = true
+
+                filterMap["Filter_" + string(i)] = filter.filter
+                filterMap["Filter_" + string(i)]["valueType"] = columns.getSpec(filter.value.column).valueType
+                
+
+                colsSpec := columns.getSpec(filter.value.column)
+                axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+                if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+                    for na, ax in colsSpec.axesSpec {
+                        if ax.name != datasetSpec.axesSpec[1].name {
+                            axisAlias := "cluster_" + string(i) + string(na)
+                            axisRenames[ax.name] = axisAlias
+                            addedAxes = append(addedAxes, ax.name)
+                        }
+                    }
+                }	
+            }
+        }
+    }
+
+
+    validRanks := false
+    if len(rankingOrder) > 0 {
+        for i, rankCol in rankingOrder {
+            if rankCol.value != undefined {
+                validRanks = true
+                col := columns.getColumn(rankCol.value.column)
+                structuredMap["Col" + string(i)] = { spec: col.spec, data: col.data }
+                addedCols = true
+
+                rankingMap["Col" + string(i)] = rankCol.rankingOrder
+                
+
+                colsSpec := columns.getSpec(rankCol.value.column)
+                axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+                if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+                    for na, ax in colsSpec.axesSpec {
+                        if ax.name != datasetSpec.axesSpec[1].name && !slices.hasElement(addedAxes, ax.name) {
+                            axisAlias := "cluster_" + string(i) + string(na)
+                            axisRenames[ax.name] = axisAlias
+                        }
+                    }
+                }	
+            }
+        }
+    }
+
+    if !validRanks {
+
+
+        if rankingOrderDefault.value != undefined {
+            i := 0
+            col := columns.getColumn(rankingOrderDefault.value.column)
+            structuredMap["Col" + string(i)] = { spec: col.spec, data: col.data }
+            addedCols = true
+
+            rankingMap["Col" + string(i)] = rankingOrderDefault.rankingOrder
+        
+
+            colsSpec := columns.getSpec(rankingOrderDefault.value.column)
+            axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+            if !slices.hasElement(axesNames, datasetSpec.axesSpec[1].name) {
+                for na, ax in colsSpec.axesSpec {
+                    if ax.name != datasetSpec.axesSpec[1].name {
+                        axisAlias := "cluster_" + string(i) + string(na)
+                        axisRenames[ax.name] = axisAlias
+                    }
+                }
+            }	
+        }
+    }
+
+
+    if len(columns.getColumns("linkers")) > 0 {
+        for i, col in columns.getColumns("linkers") {
+            if datasetSpec.axesSpec[1].name == col.spec.axesSpec[1].name {
+                structuredMap["linker." + string(i)] = { spec: col.spec, data: col.data }
+                axisAlias := "cluster_" + string(i)
+                axisRenames[col.spec.axesSpec[0].name] = axisAlias
+                linkerAxisSpec[axisAlias] = col.spec.axesSpec[0]
+            } else if datasetSpec.axesSpec[1].name == col.spec.axesSpec[0].name {
+                structuredMap["linker." + string(i)] = { spec: col.spec, data: col.data }
+                axisAlias := "cluster_" + string(i)
+                axisRenames[col.spec.axesSpec[1].name] = axisAlias
+                linkerAxisSpec[axisAlias] = col.spec.axesSpec[1]
+            }
+            addedCols = true
+        }
+    }
+
+
+    if len(columns.getColumns("clusterSizes")) > 0 {
+        for i, col in columns.getColumns("clusterSizes") {
+            structuredMap["clusterSize." + string(i)] = { spec: col.spec, data: col.data }
+            addedCols = true
+
+            for axisIdx, axis in col.spec.axesSpec {
+                if axis.name != datasetSpec.axesSpec[1].name {
+                    axisAlias := "clusterAxis_" + string(i) + "_" + string(axisIdx)
+                    axisRenames[axis.name] = axisAlias
+                }
+            }
+        }
+    }
+    
+    return {
+        structuredMap: structuredMap,
+        axisRenames: axisRenames,
+        filterMap: filterMap,
+        rankingMap: rankingMap,
+        addedCols: addedCols,
+        linkerAxisSpec: linkerAxisSpec
+    }
+}
+
+
+prepareCdr3Data := func(columns, datasetSpec, isSingleCell) {
+    cdr3SeqStructuredMap := {}
+    cdr3SeqAxisRenames := {}
+    cdr3SeqAxisRenames[datasetSpec.axesSpec[1].name] = "clonotypeKey"
+
+
+    cdr3Sequences := columns.getColumns("cdr3Sequences")
+
+    for col in cdr3Sequences {
+        headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
+        if isSingleCell {
+            if col.spec.domain["pl7.app/vdj/scClonotypeChain/index"] == "primary" {
+                cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+            }
+        } else {
+            cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+        }
+    }
+
+
+    vGenes := columns.getColumns("VGenes")	
+
+    for col in vGenes {
+        headerName := makeHeaderName(col, "vGene", isSingleCell)
+        cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+    }
+
+
+    jGenes := columns.getColumns("JGenes")	
+
+    for col in jGenes {
+        headerName := makeHeaderName(col, "jGene", isSingleCell)
+        cdr3SeqStructuredMap[headerName] = { spec: col.spec, data: col.data }
+    }
+    
+    return {
+        structuredMap: cdr3SeqStructuredMap,
+        axisRenames: cdr3SeqAxisRenames
+    }
+}
+
+
+
+filterClonotypes := func(structuredMap, axisRenames, filterMap, datasetSpec) {
+
+    buildFilterPredicate := func(columnName, filterSpec) {
+        filterType := filterSpec["type"]
+        referenceValue := filterSpec["reference"]
+        col := pt.col(columnName)
+        
+        if filterType == "number_greaterThan" {
+            return col.gt(referenceValue)
+        } else if filterType == "number_greaterThanOrEqualTo" {
+            return col.ge(referenceValue)
+        } else if filterType == "number_lessThan" {
+            return col.lt(referenceValue)
+        } else if filterType == "number_lessThanOrEqualTo" {
+            return col.le(referenceValue)
+        } else if filterType == "number_equals" {
+            return col.eq(referenceValue)
+        } else if filterType == "number_notEquals" {
+            return col.neq(referenceValue)
+        } else if filterType == "string_equals" {
+            return col.eq(string(referenceValue))
+        } else if filterType == "string_notEquals" {
+            return col.neq(string(referenceValue))
+        } else if filterType == "string_contains" {
+            return col.strContains(string(referenceValue), {literal: true})
+        } else if filterType == "string_doesNotContain" {
+            return col.strContains(string(referenceValue), {literal: true}).not()
+        }
+        ll.panic("Unknown filter type: %s", filterType)
+        return undefined
+    }
+
+
+    projection := []
+
+    projection = append(projection, pt.axis(datasetSpec.axesSpec[1].name).alias("clonotypeKey"))
+
+    for origAxis, aliasName in axisRenames {
+        projection = append(projection, pt.axis(origAxis).alias(aliasName))
+    }
+
+    for colName, _ in structuredMap {
+        projection = append(projection, pt.col(colName))
+    }
+
+
+    wfPt := pt.workflow().cacheInputs(24 * 60 * 60 * 1000)
+    
+
+    dfPt := wfPt.frame(pt.p.full(structuredMap)).select(projection...)
+
+
+
+    filterPredicates := []
+    for filterColName, filterSpec in filterMap {
+        filterType := filterSpec["type"]
+        valueType := filterSpec["valueType"]
+        
+
+        isValidFilter := false
+        if valueType == "String" && text.has_prefix(filterType, "string_") {
+            isValidFilter = true
+        } else if valueType != "String" && text.has_prefix(filterType, "number_") {
+            isValidFilter = true
+        }
+        
+        if isValidFilter {
+            predicate := buildFilterPredicate(filterColName, filterSpec)
+            filterPredicates = append(filterPredicates, predicate)
+        }
+    }
+    
+    if len(filterPredicates) > 0 {
+        dfPt = dfPt.filter(filterPredicates...)
+    }
+
+
+    dfPt = dfPt.withColumns(pt.lit(1).alias("top"))
+
+
+    frameParams := {
+        axes: [{
+            column: "clonotypeKey",
+            spec: datasetSpec.axesSpec[1]
+        }],
+        columns: [{
+            column: "top",
+            spec: {
+                name: "pl7.app/vdj/sampling-column",
+                valueType: "Int",
+                domain: {},
+                annotations: {
+                    "pl7.app/label": "Sampling column",
+                    "pl7.app/table/visibility": "optional",
+                    "pl7.app/isSubset": "true"
+                }
+            }
+        }]
+    }
+    
+    dfPt.save("filteredClonotypes.parquet")
+    dfPt.saveFrameDirect("filteredClonotypes", frameParams)
+    wfPtResult := wfPt.run()
+    
+    return {
+        filteredParquet: wfPtResult.getFile("filteredClonotypes.parquet"),
+        pframe: wfPtResult.getFrameDirect("filteredClonotypes")
+    }
+}
+
+export ll.toStrict({
+    makeHeaderName: makeHeaderName,
+    prepareClonotypeData: prepareClonotypeData,
+    prepareCdr3Data: prepareCdr3Data,
+    filterClonotypes: filterClonotypes
+})
+

package/dist/tengo/lib/{pf-kabat-conv.lib.tengo → libs.pf-kabat-conv.lib.tengo}

@@ -19,7 +19,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
         annotations: {
           "pl7.app/label": "KABAT sequence " + featureName + " Heavy",
           "pl7.app/table/orderPriority": "10",
-		      "pl7.app/table/visibility": "default"
+	      "pl7.app/table/visibility": "default"
         }
       }
     },
@@ -34,7 +34,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
         annotations: {
           "pl7.app/label": "KABAT positions " + featureName + " Heavy",
           "pl7.app/table/orderPriority": "9",
-		      "pl7.app/table/visibility": "optional"
+	      "pl7.app/table/visibility": "optional"
         }
       }
     }
@@ -53,7 +53,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
           annotations: {
             "pl7.app/label": "KABAT sequence " + featureName + " Light",
             "pl7.app/table/orderPriority": "8",
-		        "pl7.app/table/visibility": "default"
+	        "pl7.app/table/visibility": "default"
           }
         }
       },
@@ -68,7 +68,7 @@ getColumns := func(datasetSpec, featureName, bulkChain) {
           annotations: {
             "pl7.app/label": "KABAT positions " + featureName + " Light",
             "pl7.app/table/orderPriority": "7",
-		        "pl7.app/table/visibility": "optional"
+	        "pl7.app/table/visibility": "optional"
           }
         }
       }
@@ -128,4 +128,3 @@ export ll.toStrict({
   getColumns: getColumns
 })
 
-

package/dist/tengo/lib/{pf-spectratype-conv.lib.tengo → libs.pf-spectratype-conv.lib.tengo}

@@ -57,3 +57,4 @@ getColumns := func() {
 export ll.toStrict({
     getColumns: getColumns
 })
+

package/{src/pf-vj-usage-conv.lib.tengo → dist/tengo/lib/libs.pf-vj-usage-conv.lib.tengo}

@@ -52,3 +52,4 @@ getColumns := func() {
 export ll.toStrict({
     getColumns: getColumns
 })
+

package/dist/tengo/lib/{sampled-cols-conv.lib.tengo → libs.sampled-cols-conv.lib.tengo}

@@ -48,3 +48,4 @@ getColumns := func(datasetSpec, addRanking) {
 export ll.toStrict({
 	getColumns: getColumns
 })
+

package/package.json

@@ -1,20 +1,20 @@
 {
   "name": "@platforma-open/milaboratories.top-antibodies.workflow",
-  "version": "1.13.1",
+  "version": "1.14.0",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "^5.5.9",
+    "@platforma-sdk/workflow-tengo": "^5.6.4",
     "@platforma-open/milaboratories.software-anarci": "^0.0.3",
-    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.6.1",
-    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.6.0",
-    "@platforma-open/milaboratories.top-antibodies.umap": "1.1.4",
-    "@platforma-open/milaboratories.top-antibodies.assembling-fasta": "1.1.0",
-    "@platforma-open/milaboratories.top-antibodies.anarci-kabat": "1.1.0"
+    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.7.0",
+    "@platforma-open/milaboratories.top-antibodies.umap": "1.2.0",
+    "@platforma-open/milaboratories.top-antibodies.anarci-kabat": "1.2.0",
+    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.7.0",
+    "@platforma-open/milaboratories.top-antibodies.assembling-fasta": "1.2.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.3.3",
-    "@platforma-sdk/test": "^1.45.11",
+    "@platforma-sdk/tengo-builder": "^2.3.14",
+    "@platforma-sdk/test": "^1.46.3",
     "vitest": "^2.1.8"
   },
   "scripts": {

package/src/assembling-fasta.tpl.tengo

@@ -12,7 +12,7 @@ self.body(func(inputs) {
 
     inputTsv := inputs.inputTsv
     keyColumn := inputs.keyColumn // "clonotypeKey" or "scClonotypeKey"
-    finalClonotypesCsv := inputs.finalClonotypesCsv // optional
+    finalClonotypesParquet := inputs.finalClonotypesParquet // optional
     isSingleCell := inputs.isSingleCell // boolean
     bulkChain := inputs.bulkChain // "H" or "KL" when !isSingleCell
 
@@ -25,14 +25,13 @@ self.body(func(inputs) {
         arg("--key_column").arg(keyColumn).
         arg("--output_fasta").arg("assembling.fasta")
 
-    if finalClonotypesCsv != undefined {
-        cmd = cmd.addFile("finalClonotypes.csv", finalClonotypesCsv).
-            arg("--final_clonotypes_csv").arg("finalClonotypes.csv")
+    if finalClonotypesParquet != undefined {
+        cmd = cmd.addFile("finalClonotypes.parquet", finalClonotypesParquet).
+            arg("--final_clonotypes_parquet").arg("finalClonotypes.parquet")
     }
 
     cmd = cmd.saveFile("assembling.fasta").
         printErrStreamToStdout().
-        saveStdoutContent().
         cache(24 * 60 * 60 * 1000).
         run()
     
@@ -52,7 +51,6 @@ self.body(func(inputs) {
     }
     anarciBuilder = anarciBuilder.
         printErrStreamToStdout().
-        saveStdoutContent().
         cache(24 * 60 * 60 * 1000).
         run()
 
@@ -77,7 +75,6 @@ self.body(func(inputs) {
         arg("--out_tsv").arg("kabat.tsv").
         saveFile("kabat.tsv").
         printErrStreamToStdout().
-        saveStdoutContent().
         cache(24 * 60 * 60 * 1000).
         run()