@platforma-open/milaboratories.top-antibodies.workflow 1.15.1 → 1.17.0

package/.turbo/turbo-build.log

@@ -1,6 +1,6 @@
  WARN  Issue while reading "/home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.top-antibodies.workflow@1.15.1 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
+> @platforma-open/milaboratories.top-antibodies.workflow@1.17.0 build /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/assembling-fasta.tpl.tengo"...
@@ -9,17 +9,17 @@ Processing "src/main.tpl.tengo"...
 Processing "src/pf-kabat-conv.lib.tengo"...
 Processing "src/pf-spectratype-conv.lib.tengo"...
 Processing "src/pf-vj-usage-conv.lib.tengo"...
-Processing "src/prerun.tpl.tengo"...
 Processing "src/sampled-cols-conv.lib.tengo"...
+Processing "src/utils.lib.tengo"...
 No syntax errors found.
   info: Compiling 'dist'...
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/pf-kabat-conv.lib.tengo
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/pf-spectratype-conv.lib.tengo
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/pf-vj-usage-conv.lib.tengo
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/sampled-cols-conv.lib.tengo
+  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/lib/utils.lib.tengo
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/assembling-fasta.plj.gz
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/filter-and-sample.plj.gz
-  info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/prerun.plj.gz
   info:   - writing /home/runner/work/antibody-tcr-lead-selection/antibody-tcr-lead-selection/workflow/dist/tengo/tpl/main.plj.gz
   info: Template Pack build done.
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @platforma-open/milaboratories.top-antibodies.workflow
 
+## 1.17.0
+
+### Minor Changes
+
+- b201aaf: Improve cluster ranking, improve performance
+
+### Patch Changes
+
+- Updated dependencies [b201aaf]
+  - @platforma-open/milaboratories.top-antibodies.anarci-kabat@1.3.0
+  - @platforma-open/milaboratories.top-antibodies.assembling-fasta@1.3.0
+  - @platforma-open/milaboratories.top-antibodies.sample-clonotypes@1.9.0
+  - @platforma-open/milaboratories.top-antibodies.spectratype@1.8.0
+
+## 1.16.0
+
+### Minor Changes
+
+- 4ecbe6b: Improve cluster-based ranking
+
+### Patch Changes
+
+- Updated dependencies [4ecbe6b]
+  - @platforma-open/milaboratories.top-antibodies.sample-clonotypes@1.8.0
+
 ## 1.15.1
 
 ### Patch Changes

package/dist/index.cjs

@@ -1,6 +1,5 @@
 module.exports = { Templates: {
   'assembling-fasta': { type: 'from-file', path: require.resolve('./tengo/tpl/assembling-fasta.plj.gz') },
   'filter-and-sample': { type: 'from-file', path: require.resolve('./tengo/tpl/filter-and-sample.plj.gz') },
-  'prerun': { type: 'from-file', path: require.resolve('./tengo/tpl/prerun.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "assembling-fasta" | "filter-and-sample" | "prerun" | "main";
+declare type TplName = "assembling-fasta" | "filter-and-sample" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -2,6 +2,5 @@ import { resolve } from 'node:path';
 export const Templates = {
   'assembling-fasta': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/assembling-fasta.plj.gz') },
   'filter-and-sample': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/filter-and-sample.plj.gz') },
-  'prerun': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/prerun.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/dist/tengo/lib/sampled-cols-conv.lib.tengo

@@ -11,7 +11,7 @@ getColumns := func(datasetSpec, addRanking) {
           domain: {},
           annotations: {
             "pl7.app/label": "Sampling column",
-            "pl7.app/table/visibility": "optional",
+            "pl7.app/table/visibility": "hidden",
             "pl7.app/isSubset": "true"
           }
         }

package/dist/tengo/lib/utils.lib.tengo

@@ -0,0 +1,139 @@
+
+
+slices := import("@platforma-sdk/workflow-tengo:slices")
+
+
+
+
+
+
+
+
+
+clusterAxisDomainsMatch := func(axis1, axis2) {
+
+    if is_undefined(axis1) || is_undefined(axis2) {
+        return false
+    }
+    
+
+    if is_undefined(axis1.domain) != is_undefined(axis2.domain) {
+        return false
+    }
+    
+
+    if is_undefined(axis1.domain) && is_undefined(axis2.domain) {
+        return true
+    }
+    
+
+    if len(axis1.domain) != len(axis2.domain) {
+        return false
+    }
+    
+
+    for k, v in axis1.domain {
+        if is_undefined(axis2.domain[k]) || axis2.domain[k] != v {
+            return false
+        }
+    }
+    
+    return true
+}
+
+
+
+
+
+
+
+
+
+findMatchingLinkerIndex := func(colsSpec, linkerColumns) {
+
+    rankingClusterIdAxis := undefined
+    for axis in colsSpec.axesSpec {
+        if axis.name == "pl7.app/vdj/clusterId" {
+            rankingClusterIdAxis = axis
+            break
+        }
+    }
+    
+
+    if is_undefined(rankingClusterIdAxis) {
+        return undefined
+    }
+    
+    for li, linkerCol in linkerColumns {
+
+        linkerClusterIdAxis := undefined
+        for axis in linkerCol.spec.axesSpec {
+            if axis.name == "pl7.app/vdj/clusterId" {
+                linkerClusterIdAxis = axis
+                break
+            }
+        }
+        
+
+        if clusterAxisDomainsMatch(rankingClusterIdAxis, linkerClusterIdAxis) {
+            return li
+        }
+    }
+    
+    return undefined
+}
+
+
+
+
+
+
+
+
+
+
+processRankingColumn := func(colsSpec, datasetMainAxisName, linkerColumns, clusterPropertyIdx) {
+    axesNames := slices.map(colsSpec.axesSpec, func (a) { return a.name})
+    isClusterProperty := !slices.hasElement(axesNames, datasetMainAxisName)
+    
+    if !isClusterProperty {
+
+        return {
+            isClusterProperty: false,
+            header: undefined,
+            clusterAxisIdx: undefined,
+            newClusterPropertyIdx: clusterPropertyIdx
+        }
+    }
+    
+
+    linkerIdx := findMatchingLinkerIndex(colsSpec, linkerColumns)
+    
+    header := ""
+    clusterAxisIdx := undefined
+    newClusterPropertyIdx := clusterPropertyIdx
+    
+    if linkerIdx != undefined {
+
+        header = "Col_linker." + string(linkerIdx)
+        clusterAxisIdx = linkerIdx
+    } else {
+
+        header = "Col_cluster." + string(clusterPropertyIdx)
+        clusterAxisIdx = clusterPropertyIdx
+        newClusterPropertyIdx = clusterPropertyIdx + 1
+    }
+    
+    return {
+        isClusterProperty: true,
+        header: header,
+        clusterAxisIdx: clusterAxisIdx,
+        newClusterPropertyIdx: newClusterPropertyIdx
+    }
+}
+
+export {
+    clusterAxisDomainsMatch: clusterAxisDomainsMatch,
+    findMatchingLinkerIndex: findMatchingLinkerIndex,
+    processRankingColumn: processRankingColumn
+}

package/package.json

@@ -1,20 +1,20 @@
 {
   "name": "@platforma-open/milaboratories.top-antibodies.workflow",
-  "version": "1.15.1",
+  "version": "1.17.0",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "^5.6.6",
+    "@platforma-sdk/workflow-tengo": "^5.7.3",
     "@platforma-open/milaboratories.software-anarci": "^0.0.3",
-    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.7.2",
-    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.7.1",
+    "@platforma-open/milaboratories.top-antibodies.sample-clonotypes": "1.9.0",
     "@platforma-open/milaboratories.top-antibodies.umap": "1.2.1",
-    "@platforma-open/milaboratories.top-antibodies.assembling-fasta": "1.2.1",
-    "@platforma-open/milaboratories.top-antibodies.anarci-kabat": "1.2.1"
+    "@platforma-open/milaboratories.top-antibodies.assembling-fasta": "1.3.0",
+    "@platforma-open/milaboratories.top-antibodies.spectratype": "1.8.0",
+    "@platforma-open/milaboratories.top-antibodies.anarci-kabat": "1.3.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.4.1",
-    "@platforma-sdk/test": "^1.47.6",
+    "@platforma-sdk/tengo-builder": "^2.4.2",
+    "@platforma-sdk/test": "^1.48.8",
     "vitest": "^2.1.8"
   },
   "scripts": {

package/src/assembling-fasta.tpl.tengo

@@ -12,7 +12,7 @@ self.body(func(inputs) {
 
     inputTsv := inputs.inputTsv
     keyColumn := inputs.keyColumn // "clonotypeKey" or "scClonotypeKey"
-    finalClonotypesCsv := inputs.finalClonotypesCsv // optional
+    finalClonotypes := inputs.finalClonotypes // optional
     isSingleCell := inputs.isSingleCell // boolean
     bulkChain := inputs.bulkChain // "H" or "KL" when !isSingleCell
 
@@ -20,14 +20,14 @@ self.body(func(inputs) {
         software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.assembling-fasta:main")).
         cpu(1).
         mem("4GiB").
-        addFile("assembling.tsv", inputTsv).
-        arg("--input_tsv").arg("assembling.tsv").
+        addFile("assembling.parquet", inputTsv).
+        arg("--input_parquet").arg("assembling.parquet").
         arg("--key_column").arg(keyColumn).
         arg("--output_fasta").arg("assembling.fasta")
 
-    if finalClonotypesCsv != undefined {
-        cmd = cmd.addFile("finalClonotypes.csv", finalClonotypesCsv).
-            arg("--final_clonotypes_csv").arg("finalClonotypes.csv")
+    if finalClonotypes != undefined {
+        cmd = cmd.addFile("finalClonotypes.parquet", finalClonotypes).
+            arg("--final-clonotypes").arg("finalClonotypes.parquet")
     }
 
     cmd = cmd.saveFile("assembling.fasta").

package/src/filter-and-sample.tpl.tengo

@@ -8,7 +8,7 @@ render := import("@platforma-sdk/workflow-tengo:render")
 sampledColsConv := import(":sampled-cols-conv")
 json := import("json")
 
-self.defineOutputs("sampledRows", "finalClonotypesCsv")
+self.defineOutputs("sampledRows", "finalClonotypes")
 
 self.body(func(inputs) {
 
@@ -19,63 +19,73 @@ self.body(func(inputs) {
     topClonotypes := inputs.topClonotypes
 
     outputs := {}
-    finalClonotypesCsv := undefined
+    finalClonotypes := undefined
     
     // Run filtering script
     filterResult := exec.builder().
         software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.sample-clonotypes:filter")).
         mem("16GiB").
         cpu(1).
-        addFile("clonotypes.csv", cloneTable).
-        arg("--csv").arg("clonotypes.csv").
-        arg("--out").arg("filteredClonotypes.csv").
+        addFile("clonotypes.parquet", cloneTable).
+        arg("--parquet").arg("clonotypes.parquet").
+        arg("--out").arg("filteredClonotypes.parquet").
         arg("--filter-map").arg(string(json.encode(filterMap))).
-        saveFile("filteredClonotypes.csv").
+        saveFile("filteredClonotypes.parquet").
         printErrStreamToStdout().
         cache(24 * 60 * 60 * 1000).
         run()
 
-    // Save filtered CSV file
-    filteredClonotypesCsv := filterResult.getFile("filteredClonotypes.csv")
+    // Save filtered parquet file
+    filteredClonotypes := filterResult.getFile("filteredClonotypes.parquet")
 
     // Store outputs 
     sampledColsParams := sampledColsConv.getColumns(datasetSpec, false) // No ranking column
-    filteredClonotypesPf := xsv.importFile(filteredClonotypesCsv, "csv", sampledColsParams,
+    filteredClonotypesPf := xsv.importFile(filteredClonotypes, "parquet", sampledColsParams,
                                         {cpu: 1, mem: "16GiB"})
 
     // Prepare outputs in case there is no top ranking
     outputs["sampledRows"] = pframes.exportFrame(filteredClonotypesPf)
-    finalClonotypesCsv = filteredClonotypesCsv
+    finalClonotypes = filteredClonotypes
 
 	if topClonotypes != undefined {
 
 		////////// Top Clonotypes Sampling //////////
 		// Run sampling script on filtered data
-		sampleClones := exec.builder().
+		builder := exec.builder().
 			software(assets.importSoftware("@platforma-open/milaboratories.top-antibodies.sample-clonotypes:main")).
 			mem("16GiB").
 			cpu(1).
-			addFile("filteredClonotypes.csv", filteredClonotypesCsv).
-			arg("--csv").arg("filteredClonotypes.csv").
+			addFile("filteredClonotypes.parquet", filteredClonotypes).
+			arg("--parquet").arg("filteredClonotypes.parquet").
 			arg("--n").arg(string(topClonotypes)).
-			arg("--ranking-map").arg(string(json.encode(rankingMap))).
-			arg("--out").arg("sampledClonotypes_top.csv").
-			saveFile("sampledClonotypes_top.csv").
+			arg("--ranking-map").arg(string(json.encode(rankingMap)))
+
+		// Add optional parameters if provided in block args
+		if inputs.disableClusterRanking {
+			builder = builder.arg("--disable-cluster-ranking")
+		}
+		if inputs.clusterColumn != undefined && inputs.clusterColumn != "" {
+			builder = builder.arg("--cluster-column").arg(inputs.clusterColumn)
+		}
+
+		sampleClones := builder.
+			arg("--out").arg("sampledClonotypes_top.parquet").
+			saveFile("sampledClonotypes_top.parquet").
 			printErrStreamToStdout().
 			cache(24 * 60 * 60 * 1000).
 			run()
 
-		// Save top clonotypes CSV file
-		finalClonotypesCsv = sampleClones.getFile("sampledClonotypes_top.csv")
+		// Save top clonotypes parquet file
+		finalClonotypes = sampleClones.getFile("sampledClonotypes_top.parquet")
 		
 		// Store outputs 
         sampledColsParams := sampledColsConv.getColumns(datasetSpec, true) // Add ranking column
-		sampledColumnsPf := xsv.importFile(finalClonotypesCsv, "csv", sampledColsParams,
+		sampledColumnsPf := xsv.importFile(finalClonotypes, "parquet", sampledColsParams,
 											{cpu: 1, mem: "16GiB"})
 		outputs["sampledRows"] = pframes.exportFrame(sampledColumnsPf)
 	} 
 
-    outputs["finalClonotypesCsv"] = finalClonotypesCsv
+    outputs["finalClonotypes"] = finalClonotypes
 
     return outputs
 })