@platforma-open/milaboratories.top-antibodies.workflow 1.11.1 → 1.11.3

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.top-antibodies.workflow@1.11.1 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
+> @platforma-open/milaboratories.top-antibodies.workflow@1.11.3 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/filter-and-sample.tpl.tengo"...

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @platforma-open/milaboratories.top-antibodies.workflow
 
+## 1.11.3
+
+### Patch Changes
+
+- 44895be: Support parquet format
+
+## 1.11.2
+
+### Patch Changes
+
+- 65e8749: Minor bugs correction and SDK update
+
 ## 1.11.1
 
 ### Patch Changes

package/dist/tengo/lib/pf-spectratype-conv.lib.tengo

@@ -49,7 +49,7 @@ getColumns := func() {
                 }
             }
         ],
-        storageFormat: "Binary",
+        storageFormat: "Parquet",
         partitionKeyLength: 0
     }
 }

package/dist/tengo/lib/pf-vj-usage-conv.lib.tengo

@@ -44,7 +44,7 @@ getColumns := func() {
                 }
             }
         ],
-        storageFormat: "Binary",
+        storageFormat: "Parquet",
         partitionKeyLength: 0
     }
 }

package/dist/tengo/lib/sampled-cols-conv.lib.tengo

@@ -40,7 +40,7 @@ getColumns := func(datasetSpec, addRanking) {
         spec: datasetSpec.axesSpec[1]
       }],
     columns: columns,
-    storageFormat: "Binary",
+    storageFormat: "Parquet",
     partitionKeyLength: 0
   }
 }

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@platforma-open/milaboratories.top-antibodies.workflow",
-  "version": "1.11.1",
+  "version": "1.11.3",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "^5.3.3",
+    "@platforma-sdk/workflow-tengo": "^5.4.2",
     "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.4.4",
     "@platforma-open/milaboratories.top-antibodies.spectratype": "1.4.4",
     "@platforma-open/milaboratories.top-antibodies.umap": "1.1.4"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.3.0",
-    "@platforma-sdk/test": "^1.44.7",
+    "@platforma-sdk/tengo-builder": "^2.3.2",
+    "@platforma-sdk/test": "^1.44.19",
     "vitest": "^2.1.8"
   },
   "scripts": {

package/src/pf-spectratype-conv.lib.tengo

@@ -49,7 +49,7 @@ getColumns := func() {
                 }
             }
         ],
-        storageFormat: "Binary",
+        storageFormat: "Parquet",
         partitionKeyLength: 0
     }
 }

package/src/pf-vj-usage-conv.lib.tengo

@@ -44,7 +44,7 @@ getColumns := func() {
                 }
             }
         ],
-        storageFormat: "Binary",
+        storageFormat: "Parquet",
         partitionKeyLength: 0
     }
 }

package/src/prerun.tpl.tengo

@@ -125,12 +125,14 @@ wf.body(func(args) {
         addedAxes := []
         filterMap := {}
         rankingMap := {}
+        addedCols := false
         if len(args.filters) > 0 {
             for i, filter in args.filters {
                 if filter.value != undefined {
                     // Columns added here might also be in ranking list, so we add default IDs
                     cloneTable.add(columns.getColumn(filter.value.column), 
                                     {header: "Filter_" + string(i), id: "filter_" + string(i)})
+                    addedCols = true
                     // Store reference value and filter type associated to this column
                     filterMap["Filter_" + string(i)] = filter.filter
                 
@@ -156,6 +158,7 @@ wf.body(func(args) {
                 if col.value != undefined {
                     validRanks = true
                     cloneTable.add(columns.getColumn(col.value.column), {header: "Col" + string(i)})
+                    addedCols = true
                     // Store ranking order for this column
                     rankingMap["Col" + string(i)] = col.rankingOrder
                 
@@ -179,6 +182,7 @@ wf.body(func(args) {
             if args.rankingOrderDefault.value != undefined {
                 i := 0
                 cloneTable.add(columns.getColumn(args.rankingOrderDefault.value.column), {header: "Col" + string(i)})
+                addedCols = true
                 // Store default ranking order
                 rankingMap["Col" + string(i)] = args.rankingOrderDefault.rankingOrder
             
@@ -208,6 +212,7 @@ wf.body(func(args) {
                     cloneTable.setAxisHeader(col.spec.axesSpec[1].name, "cluster_" + string(i))
                     linkerAxisSpec["cluster_" + string(i)] = col.spec.axesSpec[1]
                 }
+                addedCols = true
             }
         }
 
@@ -215,6 +220,7 @@ wf.body(func(args) {
         if len(columns.getColumns("clusterSizes")) > 0 {
             for i, col in columns.getColumns("clusterSizes") {
                 cloneTable.add(col, {header: "clusterSize." + string(i)})
+                addedCols = true
                 // Add the cluster axis header
                 for axisIdx, axis in col.spec.axesSpec {
                     if axis.name != datasetSpec.axesSpec[1].name {
@@ -224,127 +230,132 @@ wf.body(func(args) {
             }
         }
 
-        cloneTable.mem("16GiB")
-        cloneTable.cpu(1)
-        cloneTable = cloneTable.build()
-
-        // Use ender.create to call the filter-clonotypes template
-        filterSampleResult := render.create(filterAndSampleTpl, {
-            inputAnchor: args.inputAnchor,
-            cloneTable: cloneTable,
-            rankingOrder: args.rankingOrder,
-            rankingOrderDefault: args.rankingOrderDefault,
-            filters: args.filters,
-            filterMap: filterMap,
-            rankingMap: rankingMap,
-            datasetSpec: datasetSpec,
-            topClonotypes: args.topClonotypes
-        })
-        
-        // Get the filtered clonotypes from the template result
-        outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
-
-        // Get the filtered and sampled clonotypes P-frame and CSV from the template result
-        finalClonotypesCsv := filterSampleResult.output("finalClonotypesCsv", 24 * 60 * 60 * 1000)
-        // outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
-        
-        ////////// CDR3 Length Calculation //////////
-        
-        cdr3SeqTable := pframes.tsvFileBuilder()
-        cdr3SeqTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
-
-        // Must deal with multiple CDR3 sequences (two for each cell in single cell data)
-        // Chain will be added in the header as cdr3Sequence.chain and used in python script
-        // Notice chain is in spec.domain for single cell data and spec.axesSpec[0].domain for bulk data
-
-        // Helper function to add chain information to the headers dynamically
-        chainMapping := {
-            "IG": { "A": "Heavy", "B": "Light" },
-            "TCRAB": { "A": "TRA", "B": "TRB" },
-            "TCRGD": { "A": "TRG", "B": "TRD" }
-        }
+        // Continue only if we have at least a column
+        // This condition prevents temporal intermittent error while filters are 
+        // being processed and possibly in other situations too
+        if addedCols {
+            cloneTable.mem("16GiB")
+            cloneTable.cpu(1)
+            cloneTable = cloneTable.build()
+
+            // Use ender.create to call the filter-clonotypes template
+            filterSampleResult := render.create(filterAndSampleTpl, {
+                inputAnchor: args.inputAnchor,
+                cloneTable: cloneTable,
+                rankingOrder: args.rankingOrder,
+                rankingOrderDefault: args.rankingOrderDefault,
+                filters: args.filters,
+                filterMap: filterMap,
+                rankingMap: rankingMap,
+                datasetSpec: datasetSpec,
+                topClonotypes: args.topClonotypes
+            })
+            
+            // Get the filtered clonotypes from the template result
+            outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
 
-        makeHeaderName := func(col, baseHeaderName, isSingleCell) {
-            if isSingleCell {
-                chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
-                receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
-                chainLabel := chainMapping[receptor][chain]
-                return baseHeaderName + "." + chainLabel // e.g., "cdr3Sequence.Heavy"
-            } else {
-                // For bulk, if chain info is available (e.g. IGH, IGK, IGL)
-                chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
-                if chainFromDomain != undefined {
-                    return baseHeaderName + "." + chainFromDomain // e.g., "cdr3Sequence.IGH"
-                }
+            // Get the filtered and sampled clonotypes P-frame and CSV from the template result
+            finalClonotypesCsv := filterSampleResult.output("finalClonotypesCsv", 24 * 60 * 60 * 1000)
+            // outputs["sampledRows"] = filterSampleResult.output("sampledRows", 24 * 60 * 60 * 1000)
+            
+            ////////// CDR3 Length Calculation //////////
+            
+            cdr3SeqTable := pframes.tsvFileBuilder()
+            cdr3SeqTable.setAxisHeader(datasetSpec.axesSpec[1].name, "clonotypeKey")
+
+            // Must deal with multiple CDR3 sequences (two for each cell in single cell data)
+            // Chain will be added in the header as cdr3Sequence.chain and used in python script
+            // Notice chain is in spec.domain for single cell data and spec.axesSpec[0].domain for bulk data
+
+            // Helper function to add chain information to the headers dynamically
+            chainMapping := {
+                "IG": { "A": "Heavy", "B": "Light" },
+                "TCRAB": { "A": "TRA", "B": "TRB" },
+                "TCRGD": { "A": "TRG", "B": "TRD" }
             }
-            return baseHeaderName
-        };
 
-        // Process CDR3 sequences
-        cdr3Sequences := columns.getColumns("cdr3Sequences")
+            makeHeaderName := func(col, baseHeaderName, isSingleCell) {
+                if isSingleCell {
+                    chain := col.spec.domain["pl7.app/vdj/scClonotypeChain"]  // e.g., "A", "B"
+                    receptor := col.spec.axesSpec[0].domain["pl7.app/vdj/receptor"]  // e.g., "IG", "TCRAB", "TCRGD"
+                    chainLabel := chainMapping[receptor][chain]
+                    return baseHeaderName + "." + chainLabel // e.g., "cdr3Sequence.Heavy"
+                } else {
+                    // For bulk, if chain info is available (e.g. IGH, IGK, IGL)
+                    chainFromDomain := col.spec.axesSpec[0].domain["pl7.app/vdj/chain"] // e.g. "IGH", "IGK"
+                    if chainFromDomain != undefined {
+                        return baseHeaderName + "." + chainFromDomain // e.g., "cdr3Sequence.IGH"
+                    }
+                }
+                return baseHeaderName
+            };
 
-        for col in cdr3Sequences {
-            headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
-            cdr3SeqTable.add(col, {header: headerName})
-        }
+            // Process CDR3 sequences
+            cdr3Sequences := columns.getColumns("cdr3Sequences")
 
-        // Process V genes
-        vGenes := columns.getColumns("VGenes")	
+            for col in cdr3Sequences {
+                headerName := makeHeaderName(col, "cdr3Sequence", isSingleCell)
+                cdr3SeqTable.add(col, {header: headerName})
+            }
 
-        for col in vGenes {
-            headerName := makeHeaderName(col, "vGene", isSingleCell)
-            cdr3SeqTable.add(col, {header: headerName})
-        }
+            // Process V genes
+            vGenes := columns.getColumns("VGenes")	
 
-        // Process J genes
-        jGenes := columns.getColumns("JGenes")	
+            for col in vGenes {
+                headerName := makeHeaderName(col, "vGene", isSingleCell)
+                cdr3SeqTable.add(col, {header: headerName})
+            }
 
-        for col in jGenes {
-            headerName := makeHeaderName(col, "jGene", isSingleCell)
-            cdr3SeqTable.add(col, {header: headerName})
-        }
+            // Process J genes
+            jGenes := columns.getColumns("JGenes")	
 
-        cdr3SeqTable.mem("16GiB")
-        cdr3SeqTable.cpu(1)
-        cdr3SeqTableBuilt := cdr3SeqTable.build()
-
-        cdr3VspectratypeCmd := exec.builder().
-            software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.spectratype:main")).
-            mem("16GiB").
-            cpu(1).
-            addFile("cdr3_sequences_input.tsv", cdr3SeqTableBuilt).
-            arg("--input_tsv").arg("cdr3_sequences_input.tsv").
-            arg("--spectratype_tsv").arg("spectratype.tsv").
-            arg("--vj_usage_tsv").arg("vj_usage.tsv") // no dot here
-
-        // Add top clonotypes argument and file to the builder if provided
-        if finalClonotypesCsv != undefined {
-            cdr3VspectratypeCmd = cdr3VspectratypeCmd.
-                arg("--final_clonotypes_csv").arg("finalClonotypes.csv").
-                addFile("finalClonotypes.csv", finalClonotypesCsv)
-        }
+            for col in jGenes {
+                headerName := makeHeaderName(col, "jGene", isSingleCell)
+                cdr3SeqTable.add(col, {header: headerName})
+            }
+
+            cdr3SeqTable.mem("16GiB")
+            cdr3SeqTable.cpu(1)
+            cdr3SeqTableBuilt := cdr3SeqTable.build()
+
+            cdr3VspectratypeCmd := exec.builder().
+                software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.spectratype:main")).
+                mem("16GiB").
+                cpu(1).
+                addFile("cdr3_sequences_input.tsv", cdr3SeqTableBuilt).
+                arg("--input_tsv").arg("cdr3_sequences_input.tsv").
+                arg("--spectratype_tsv").arg("spectratype.tsv").
+                arg("--vj_usage_tsv").arg("vj_usage.tsv") // no dot here
+
+            // Add top clonotypes argument and file to the builder if provided
+            if finalClonotypesCsv != undefined {
+                cdr3VspectratypeCmd = cdr3VspectratypeCmd.
+                    arg("--final_clonotypes_csv").arg("finalClonotypes.csv").
+                    addFile("finalClonotypes.csv", finalClonotypesCsv)
+            }
 
-        cdr3VspectratypeCmd = cdr3VspectratypeCmd. // continue building the command
-            saveFile("spectratype.tsv").
-            saveFile("vj_usage.tsv").
-            printErrStreamToStdout().
-            saveStdoutContent().
-            cache(24 * 60 * 60 * 1000).
-            run()
+            cdr3VspectratypeCmd = cdr3VspectratypeCmd. // continue building the command
+                saveFile("spectratype.tsv").
+                saveFile("vj_usage.tsv").
+                printErrStreamToStdout().
+                saveStdoutContent().
+                cache(24 * 60 * 60 * 1000).
+                run()
 
 
-        // Spectratype PFrame structure is [chain][cdr3Length][vGene] -> count
+            // Spectratype PFrame structure is [chain][cdr3Length][vGene] -> count
 
-        cdr3VspectratypePf := xsv.importFile(cdr3VspectratypeCmd.getFile("spectratype.tsv"), 
-                                            "tsv", spectratypeConv.getColumns(),
-                                            {cpu: 1, mem: "16GiB"})
-        outputs["cdr3VspectratypePf"] = pframes.exportFrame(cdr3VspectratypePf) 
+            cdr3VspectratypePf := xsv.importFile(cdr3VspectratypeCmd.getFile("spectratype.tsv"), 
+                                                "tsv", spectratypeConv.getColumns(),
+                                                {cpu: 1, mem: "16GiB"})
+            outputs["cdr3VspectratypePf"] = pframes.exportFrame(cdr3VspectratypePf) 
 
-        // For vjUsage structure is [chain][vGene][jGene] -> count
-        vjUsagePf := xsv.importFile(cdr3VspectratypeCmd.getFile("vj_usage.tsv"), 
-                                    "tsv", vjUsageConv.getColumns(), 
-                                    {cpu: 1, mem: "16GiB"})
-        outputs["vjUsagePf"] = pframes.exportFrame(vjUsagePf)
+            // For vjUsage structure is [chain][vGene][jGene] -> count
+            vjUsagePf := xsv.importFile(cdr3VspectratypeCmd.getFile("vj_usage.tsv"), 
+                                        "tsv", vjUsageConv.getColumns(), 
+                                        {cpu: 1, mem: "16GiB"})
+            outputs["vjUsagePf"] = pframes.exportFrame(vjUsagePf)
+        }
     }
 
 	return {

package/src/sampled-cols-conv.lib.tengo

@@ -40,7 +40,7 @@ getColumns := func(datasetSpec, addRanking) {
         spec: datasetSpec.axesSpec[1]
       }],
     columns: columns,
-    storageFormat: "Binary",
+    storageFormat: "Parquet",
     partitionKeyLength: 0
   }
 }