@platforma-open/milaboratories.immune-assay-data.workflow 1.7.1 → 1.8.0

package/.turbo/turbo-build.log

@@ -1,17 +1,25 @@
  WARN  Issue while reading "/home/runner/work/immune-assay-data/immune-assay-data/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.immune-assay-data.workflow@1.7.1 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
+> @platforma-open/milaboratories.immune-assay-data.workflow@1.8.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
+Processing "src/analysis.tpl.tengo"...
 Processing "src/build-outputs.tpl.tengo"...
+Processing "src/check-content-empty.tpl.tengo"...
 Processing "src/extract-unique-values.tpl.tengo"...
+Processing "src/get-unique-values.tpl.tengo"...
 Processing "src/main.tpl.tengo"...
+Processing "src/process-outputs.tpl.tengo"...
 Processing "src/run-alignment.tpl.tengo"...
 No syntax errors found.
   info: Compiling 'dist'...
   info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/build-outputs.plj.gz
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/check-content-empty.plj.gz
   info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/extract-unique-values.plj.gz
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/get-unique-values.plj.gz
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/process-outputs.plj.gz
   info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/run-alignment.plj.gz
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/analysis.plj.gz
   info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/main.plj.gz
   info: Template Pack build done.
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @platforma-open/milaboratories.immune-assay-data.workflow
 
+## 1.8.0
+
+### Minor Changes
+
+- de02090: Allow deduplication and minor fixes
+
 ## 1.7.1
 
 ### Patch Changes

package/dist/index.cjs

@@ -1,6 +1,10 @@
 module.exports = { Templates: {
   'build-outputs': { type: 'from-file', path: require.resolve('./tengo/tpl/build-outputs.plj.gz') },
+  'check-content-empty': { type: 'from-file', path: require.resolve('./tengo/tpl/check-content-empty.plj.gz') },
   'extract-unique-values': { type: 'from-file', path: require.resolve('./tengo/tpl/extract-unique-values.plj.gz') },
+  'get-unique-values': { type: 'from-file', path: require.resolve('./tengo/tpl/get-unique-values.plj.gz') },
+  'process-outputs': { type: 'from-file', path: require.resolve('./tengo/tpl/process-outputs.plj.gz') },
   'run-alignment': { type: 'from-file', path: require.resolve('./tengo/tpl/run-alignment.plj.gz') },
+  'analysis': { type: 'from-file', path: require.resolve('./tengo/tpl/analysis.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "build-outputs" | "extract-unique-values" | "run-alignment" | "main";
+declare type TplName = "build-outputs" | "check-content-empty" | "extract-unique-values" | "get-unique-values" | "process-outputs" | "run-alignment" | "analysis" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,7 +1,11 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'build-outputs': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/build-outputs.plj.gz') },
+  'check-content-empty': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/check-content-empty.plj.gz') },
   'extract-unique-values': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/extract-unique-values.plj.gz') },
+  'get-unique-values': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/get-unique-values.plj.gz') },
+  'process-outputs': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/process-outputs.plj.gz') },
   'run-alignment': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/run-alignment.plj.gz') },
+  'analysis': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/analysis.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "@platforma-open/milaboratories.immune-assay-data.workflow",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "type": "module",
   "description": "Tengo-based template",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "5.9.0",
+    "@platforma-sdk/workflow-tengo": "5.9.1",
     "@platforma-open/soedinglab.software-mmseqs2": "1.18.3",
-    "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.3",
     "@platforma-open/milaboratories.immune-assay-data.add-header": "1.1.3",
-    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0",
+    "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.3",
     "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.2.0",
-    "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.3"
+    "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.3",
+    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "1.1.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "2.4.25"
+    "@platforma-sdk/tengo-builder": "2.4.27"
   },
   "scripts": {
     "build": "rm -rf dist && pl-tengo check && pl-tengo build",

package/src/analysis.tpl.tengo

@@ -0,0 +1,370 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+text := import("text")
+json := import("json")
+render := import("@platforma-sdk/workflow-tengo:render")
+
+prepareFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.prepare-fasta:main")
+fastaToTsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv:main")
+addHeaderSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.add-header:main")
+covModeCalcSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc:main")
+xlsxToCsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv:main")
+
+runAlignmentTpl := assets.importTemplate(":run-alignment")
+checkContentEmptyTpl := assets.importTemplate(":check-content-empty")
+getUniqueValuesTpl := assets.importTemplate(":get-unique-values")
+
+/**
+ * Convert xlsx file to csv file
+ * @param fileXlsx - xlsx file
+ * @return csv file run result
+ */
+ runXlsxToCsv := func(fileXlsx) {
+	e := exec.builder().
+		software(xlsxToCsvSw).
+		mem("16GiB").
+		cpu(1).
+		addFile("input.xlsx", fileXlsx).
+		arg("-i").arg("input.xlsx").
+		arg("-o").arg("output.csv").
+		saveFile("output.csv")
+	return e.run()
+}
+
+/**
+ * Convert fasta file to tsv file
+ * @param fileFasta - fasta file
+ * @return tsv file run result
+ */
+runFastaToTsv := func(fileFasta) {
+	e := exec.builder().
+		software(fastaToTsvSw).
+		mem("8GiB").
+		cpu(1).
+		addFile("input.fasta", fileFasta).
+		arg("-i").arg("input.fasta").
+		arg("-o").arg("output.tsv").
+		saveFile("output.tsv")
+	return e.run()
+}
+
+/**
+ * Convert tsv file to fasta file
+ * @param fileTsv - tsv file
+ * @param idCol - id column header
+ * @param seqCol - sequence column header
+ * @return fasta file run result
+ */
+runTsvToFasta := func(fileTsv, idCol, seqCol) {
+	e := exec.builder().
+		software(prepareFastaSw).
+		mem("8GiB").
+		cpu(1).
+		addFile("input.tsv", fileTsv).
+		arg("-i").arg("input.tsv").
+		arg("-o").arg("output.fasta").
+		arg("--seq_col").arg(seqCol).
+		arg("--id_col").arg(idCol).
+		saveFile("output.fasta")
+	return e.run()
+}
+
+prepareAssayFile := func(file, xsvType, sequenceColumnHeader) {
+    // assign ids to assay sequences
+    ptw := pt.workflow()
+    df := ptw.frame({
+        file: file,
+        xsvType: xsvType
+    })
+
+    //////// calculate sequence id ////////
+    // Create unique seqId for each row by combining sequence with row index
+    // First add row index using ordinal rank
+    df = df.withColumns(
+        pt.rank(pt.col(sequenceColumnHeader)).
+            over(pt.col(sequenceColumnHeader)).
+            alias("rowIndex")
+    )
+    // Concatenate sequence with row index and then hash
+    df = df.withColumns(
+        pt.when(pt.col("rowIndex").gt(pt.lit(1))).
+            then(pt.concatStr([pt.col(sequenceColumnHeader), pt.col("rowIndex").cast("String")], 
+            {delimiter: "_"})).
+            otherwise(pt.col(sequenceColumnHeader)).
+            alias("uniqueKey")
+    )
+    // Create hash from the unique key
+    df = df.addColumns(
+        pt.col("uniqueKey").hash("sha256", "base64_alphanumeric", 120).alias("seqId")
+    )
+    df = df.withoutColumns("uniqueKey", "rowIndex")
+    //////// add label to ids ////////
+    df = df.withColumns(
+        pt.col("seqId").
+            strReplace("\\d", "", { replaceAll: true }).
+            strSlice(0, 5).               // Take first 5 characters
+            strToUpper().                 // Convert to uppercase
+            alias("tmpLabel")
+    )
+    df = df.withColumns(
+        pt.rank(pt.col("seqId")).  // Rank based on clonotypeKeyCol (default ascending)
+            over(pt.col("tmpLabel")).   // Partition by prefixTempCol
+            alias("rank")
+    )
+    df = df.withColumns(
+        pt.when(pt.col("rank").gt(pt.lit(1))).
+            then(pt.concatStr([pt.lit("A"), pt.col("tmpLabel"), pt.col("rank").cast("String")], { 
+            delimiter: "-" })).
+            otherwise(pt.concatStr([pt.lit("A"), pt.col("tmpLabel")], { delimiter: "-" })).
+            alias("seqIdLabel")
+    )
+    df = df.withoutColumns("rank", "tmpLabel")
+
+    //////// add sequence column ////////
+    df = df.addColumns(
+        pt.col(sequenceColumnHeader).alias("sequence")
+    )
+    df.save("output.tsv")
+    
+    return ptw.run().getFile("output.tsv")
+}
+
+self.defineOutputs(
+	"bestAlignmentTsv",
+	"assayDataTsv",
+	"clonesDataTsv",
+	"mmseqsOutput",
+	"emptyResults",
+	"uniqueValuesMap",
+	"columnsToImport"
+)
+
+self.body(func(args) {
+	file := args.file
+	xsvType := args.xsvType
+	sequenceColumnHeader := args.sequenceColumnHeader
+	clonesFasta := args.clonesFasta
+	emptyClonesInput := args.emptyClonesInput
+	targetSequenceType := args.targetSequenceType
+	assaySequenceType := args.assaySequenceType
+	coverageThreshold := args.coverageThreshold
+	identityThreshold := args.identityThreshold
+	similarityType := args.similarityType
+	lessSensitive := args.lessSensitive
+	mem := args.metaInputs.mem
+	cpu := args.metaInputs.cpu
+
+	// Filter columns to import based on user selection
+	columnsToImport := args.importColumns
+	if args.selectedColumns != undefined && len(args.selectedColumns) > 0 {
+		selectedHeaders := {}
+		for header in args.selectedColumns {
+			selectedHeaders[header] = true
+		}
+
+		filteredColumns := []
+		for col in args.importColumns {
+			// Always include the main sequence column
+			if col.header == sequenceColumnHeader || selectedHeaders[col.header] {
+				filteredColumns = append(filteredColumns, col)
+			}
+		}
+		columnsToImport = filteredColumns
+	}
+
+	// Determine string columns for unique value extraction
+	stringColumns := []
+	for h in columnsToImport {
+		if h.type == "String" && h.header != sequenceColumnHeader {
+			stringColumns = append(stringColumns, h.header)
+		}
+	}
+
+	// Check if clones input is empty (resolved from upstream check template)
+	if emptyClonesInput {
+		return {
+			bestAlignmentTsv: {},
+			assayDataTsv: {},
+			clonesDataTsv: {},
+			mmseqsOutput: {},
+			emptyResults: true,
+			uniqueValuesMap: {},
+			columnsToImport: columnsToImport
+		}
+	}
+
+	// Handle XLSX files (convert to CSV)
+	if xsvType == "xlsx" {
+		xlsxToCsvRun := runXlsxToCsv(file)
+		file = xlsxToCsvRun.getFile("output.csv")
+		xsvType = "csv"
+	}
+
+	// Handle FASTA files (convert to TSV)
+	if xsvType == "fasta" || xsvType == "fa" {
+		fastaToTsvRun := runFastaToTsv(file)
+		file = fastaToTsvRun.getFile("output.tsv")
+		xsvType = "tsv"
+	}
+
+	assayTsv := prepareAssayFile(file, xsvType, sequenceColumnHeader)
+
+	// Prepare assay FASTA (clones FASTA already prepared upstream)
+	assayFastaRun := runTsvToFasta(assayTsv, "seqId", "sequence")
+	assayFasta := assayFastaRun.getFile("output.fasta")
+
+	// Dynamically determine coverage mode by comparing average sequence lengths
+	coverageModeRun := exec.builder().
+		software(covModeCalcSw).
+		mem("16GiB").
+		cpu(1).
+		addFile("clones.fasta", clonesFasta).
+		addFile("assay.fasta", assayFasta).
+		arg("--clones-fasta").arg("clones.fasta").
+		arg("--assay-fasta").arg("assay.fasta").
+		arg("--output").arg("coverage_mode.txt").
+		saveFileContent("coverage_mode.txt").
+		run()
+
+	covMode := coverageModeRun.getFileContent("coverage_mode.txt")
+
+	// MMseqs2 Alignment
+	mmseqsSearchType := "0"
+    if targetSequenceType == "aminoacid" && assaySequenceType == "aminoacid" {
+        //1: amino acid
+        mmseqsSearchType = "1"
+    } else if targetSequenceType == "nucleotide" && assaySequenceType == "nucleotide" {
+        // 3: nucleotide
+        mmseqsSearchType = "3"
+    } else if targetSequenceType == "nucleotide" && assaySequenceType == "aminoacid" {
+        // 4: translated nucleotide alignment
+        mmseqsSearchType = "4"
+    } else if targetSequenceType == "aminoacid" && assaySequenceType == "nucleotide"  {
+        // 2: nucleotide
+        mmseqsSearchType = "2"
+    }
+
+	runMmseqs := render.create(runAlignmentTpl, {
+		covMode: covMode,
+		mmseqsSearchType: mmseqsSearchType,
+		coverageThreshold: coverageThreshold,
+		identityThreshold: identityThreshold,
+		similarityType: similarityType,
+		clonesFasta: clonesFasta,
+		assayFasta: assayFasta,
+		lessSensitive: lessSensitive
+	}, {
+		metaInputs: {
+			mem: mem,
+			cpu: cpu
+		}
+	})
+
+	mmseqsOutput := runMmseqs.output("mmseqsOutput")
+
+	// @TODO remove header stuff and replace with pt when available (!)
+	addHeaderRun := exec.builder().
+		software(addHeaderSw).
+		mem("16GiB").
+		cpu(1).
+		arg("-i").arg("results.tsv").
+		arg("-o").arg("results_with_header.tsv").
+		addFile("results.tsv", mmseqsOutput).
+		saveFile("results_with_header.tsv").
+		saveFileContent("results_with_header.tsv").
+		run()
+
+	mmseqsResultTsv := addHeaderRun.getFile("results_with_header.tsv")
+	mmseqsResultContent := addHeaderRun.getFileContent("results_with_header.tsv")
+
+	// Use subtemplate to check if file is empty
+	checkResult := render.create(checkContentEmptyTpl, {
+		content: mmseqsResultContent,
+		mode: "headerOnly"
+	})
+	emptyResults := checkResult.output("result")
+
+	result := {
+		mmseqsOutput: mmseqsOutput,
+		emptyResults: emptyResults
+	}
+
+	// Post-processing with PT
+	ptw2 := pt.workflow()
+	dfRes := ptw2.frame({
+		file: mmseqsResultTsv,
+		xsvType: "tsv"
+	})
+
+	// Cast columns to ensure correct types for aggregation
+	dfRes = dfRes.withColumns(
+		pt.col("evalue").cast("Float64").alias("evalue"),
+		pt.col("bits").cast("Float64").alias("bits")
+	)
+
+	cols := []
+	for _, col in ["bits", "evalue", "query", "pident", "alnlen", "mismatch",
+	"gapopen", "qstart", "qend", "tstart", "tend"] {
+		cols = append(cols,
+			pt.col(col).maxBy(
+				pt.col("evalue").multiply(-1),
+				pt.col("bits")
+			).alias(col)
+		)
+	}
+
+	dfRes = dfRes.groupBy("target").agg(cols...)
+	// Add link column for linker pFrame (assayLinkerPframe)
+	dfRes = dfRes.withColumns(
+		pt.lit(1).cast("Int64").alias("link")
+	)
+	dfRes.save("best_alignment.tsv")
+
+	// Assay data summary
+	assayDf := ptw2.frame({
+		file: assayTsv,
+		xsvType: "tsv"
+	})
+
+	// import how many matches per assay sequence found
+	assayDf = assayDf.join(
+		dfRes.groupBy("query").agg(
+			pt.col("target").count().alias("queryCount")
+		),
+		{
+			how: "left",
+			leftOn: "seqId",
+			rightOn: "query"
+		}
+	)
+	assayDf.save("assay_data.tsv")
+
+	// Clones data
+	clonesDf := dfRes.join(assayDf,
+		{
+			how: "left",
+			leftOn: "query",
+			rightOn: "seqId"
+		}
+	)
+	clonesDf.save("clones_data.tsv")
+
+	ptRun2 := ptw2.run()
+
+	result.bestAlignmentTsv = ptRun2.getFile("best_alignment.tsv")
+	result.assayDataTsv = ptRun2.getFile("assay_data.tsv")
+	result.clonesDataTsv = ptRun2.getFile("clones_data.tsv")
+
+	// Extract unique values for string columns
+	getUniqueValuesResult := render.create(getUniqueValuesTpl, {
+		assayDataTsv: result.assayDataTsv,
+		stringColumns: stringColumns
+	})
+	result.uniqueValuesMap = getUniqueValuesResult.output("uniqueValuesMap")
+	result.columnsToImport = columnsToImport
+
+	return result
+})

package/src/build-outputs.tpl.tengo

@@ -57,24 +57,7 @@ self.body(func(inputs) {
 		}
 	]
 	
-	columnsToImport := inputs.importColumns
-	if inputs.selectedColumns != undefined && len(inputs.selectedColumns) > 0 {
-		selectedHeaders := {}
-		for header in inputs.selectedColumns {
-			selectedHeaders[header] = true
-		}
-		
-		filteredColumns := []
-		for col in inputs.importColumns {
-			// Always include the main sequence column
-			if col.header == inputs.sequenceColumnHeader || selectedHeaders[col.header] {
-				filteredColumns = append(filteredColumns, col)
-			}
-		}
-		columnsToImport = filteredColumns
-	}
-
-	for h in columnsToImport {
+	for h in inputs.columnsToImport {
 		if h.header == inputs.sequenceColumnHeader {
 			continue
 		}
@@ -160,7 +143,7 @@ self.body(func(inputs) {
 		}
 	}]
 
-	for h in columnsToImport {
+	for h in inputs.columnsToImport {
 		annotations := {
 			"pl7.app/label": h.header,
 			"pl7.app/table/visibility": h.header == inputs.sequenceColumnHeader ? "optional" : "default"

package/src/check-content-empty.tpl.tengo

@@ -0,0 +1,21 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+text := import("text")
+
+self.defineOutputs("result")
+
+self.body(func(args) {
+    content := string(args.content.getData())
+    mode := args.mode
+
+    isEmpty := false
+    if mode == "raw" {
+        // No header line (e.g. FASTA): empty content means no data
+        isEmpty = text.trim_space(content) == ""
+    } else if mode == "headerOnly" {
+        // File has a header line (e.g. TSV with added header): empty means <= 1 line
+        lines := text.split(text.trim_space(content), "\n")
+        isEmpty = len(lines) <= 1
+    }
+
+    return { result: isEmpty }
+})

package/src/extract-unique-values.tpl.tengo

@@ -10,8 +10,7 @@ self.body(func(inputs) {
 	uniqueValuesMap := {}
 
 	for header, contentField in fileContents {
-		// In subtemplate, we can call getData() directly on exec fileContent fields
-		// (following clonotype-enrichment pattern)
+		// In subtemplate, we can call getData() directly on inputs
 		contentBytes := contentField.getData()
 		content := string(contentBytes)
 		lines := text.split(text.trim_space(content), "\n")
@@ -20,8 +19,7 @@ self.body(func(inputs) {
 			// Skip header and collect values
 			values := lines[1:]
 			// JSON encode the array of strings and convert to string
-			encodedBytes := json.encode(values)
-			uniqueValuesMap[header] = string(encodedBytes)
+			uniqueValuesMap[header] = string(json.encode(values))
 		}
 	}
 
@@ -29,4 +27,3 @@ self.body(func(inputs) {
 		uniqueValuesMap: uniqueValuesMap
 	}
 })
-

package/src/get-unique-values.tpl.tengo

@@ -0,0 +1,51 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+render := import("@platforma-sdk/workflow-tengo:render")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+strings := import("@platforma-sdk/workflow-tengo:strings")
+
+extractUniqueValuesTpl := assets.importTemplate(":extract-unique-values")
+
+self.defineOutputs("uniqueValuesMap")
+
+self.body(func(inputs) {
+	assayDataTsv := inputs.assayDataTsv
+	stringColumns := inputs.stringColumns
+	
+	if len(stringColumns) == 0 {
+		return { uniqueValuesMap: {} }
+	}
+
+	uniqueValuesWf := pt.workflow().mem("4GiB").cpu(1)
+	baseDf := uniqueValuesWf.frame({
+		file: assayDataTsv,
+		xsvType: "tsv"
+	})
+
+	// Process each String column to extract unique values
+	for colHeader in stringColumns {
+		uniqueValuesDf := baseDf.select(pt.col(colHeader).alias("value")).groupBy("value").agg(pt.col("value").count().alias("_count"))
+		uniqueValuesDf = uniqueValuesDf.select("value")
+		fileName := "unique_values_" + strings.substituteSpecialCharacters(colHeader) + ".csv"
+		uniqueValuesDf.saveContent(fileName)
+	}
+
+	// Run once and collect all results
+	uniqueValuesResult := uniqueValuesWf.run()
+	
+	fileContentsMap := {}
+	for colHeader in stringColumns {
+		fileName := "unique_values_" + strings.substituteSpecialCharacters(colHeader) + ".csv"
+		fileContentsMap[colHeader] = uniqueValuesResult.getFileContent(fileName)
+	}
+
+	// Now call the extraction subtemplate which will use .getData() on these resources
+	extractResult := render.create(extractUniqueValuesTpl, {
+		fileContents: fileContentsMap
+	})
+
+	return {
+		uniqueValuesMap: extractResult.output("uniqueValuesMap")
+	}
+})

package/src/main.tpl.tengo

@@ -3,25 +3,19 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 file := import("@platforma-sdk/workflow-tengo:file")
 exec := import("@platforma-sdk/workflow-tengo:exec")
 assets:= import("@platforma-sdk/workflow-tengo:assets")
-maps:= import("@platforma-sdk/workflow-tengo:maps")
-xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
 pframes := import("@platforma-sdk/workflow-tengo:pframes")
-pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
 pt := import("@platforma-sdk/workflow-tengo:pt")
 path := import("@platforma-sdk/workflow-tengo:path")
 json := import("json")
 text := import("text")
 render := import("@platforma-sdk/workflow-tengo:render")
 strings := import("@platforma-sdk/workflow-tengo:strings")
-runAlignmentTpl := assets.importTemplate(":run-alignment")
-extractUniqueValuesTpl := assets.importTemplate(":extract-unique-values")
-buildOutputsTpl := assets.importTemplate(":build-outputs")
+
+analysisTpl := assets.importTemplate(":analysis")
+processOutputsTpl := assets.importTemplate(":process-outputs")
+checkContentEmptyTpl := assets.importTemplate(":check-content-empty")
 
 prepareFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.prepare-fasta:main")
-fastaToTsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv:main")
-addHeaderSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.add-header:main")
-covModeCalcSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc:main")
-xlsxToCsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv:main")
 
 wf.prepare(func(args){
 	bundleBuilder := wf.createPBundleBuilder()
@@ -32,65 +26,6 @@ wf.prepare(func(args){
 	}
 })
 
-prepareAssayFile := func(args, file, xsvType) {
-	// assign ids to assay sequences
-	ptw := pt.workflow()
-	df := ptw.frame({
-		file: file,
-		xsvType: xsvType
-	})
-
-	//////// calculate sequence id ////////
-	// Create unique seqId for each row by combining sequence with row index
-	// First add row index using ordinal rank
-	df = df.withColumns(
-		pt.rank(pt.col(args.sequenceColumnHeader)).
-			over(pt.col(args.sequenceColumnHeader)).
-			alias("rowIndex")
-	)
-	// Concatenate sequence with row index and then hash
-	df = df.withColumns(
-		pt.when(pt.col("rowIndex").gt(pt.lit(1))).
-			then(pt.concatStr([pt.col(args.sequenceColumnHeader), pt.col("rowIndex").cast("String")], {delimiter: "_"})).
-			otherwise(pt.col(args.sequenceColumnHeader)).
-			alias("uniqueKey")
-	)
-	// Create hash from the unique key
-	df = df.addColumns(
-		pt.col("uniqueKey").hash("sha256", "base64_alphanumeric", 120).alias("seqId")
-	)
-	// Remove the temporary columns
-	//df = df.withoutColumns("uniqueKey", "rowIndex")
-	//////// add label to ids ////////
-	df = df.withColumns(
-        pt.col("seqId").
-            strReplace("\\d", "", { replaceAll: true }).
-            strSlice(0, 5).               // Take first 5 characters
-            strToUpper().                 // Convert to uppercase
-            alias("tmpLabel")
-    )
-	df = df.withColumns(
-        pt.rank(pt.col("seqId")).  // Rank based on clonotypeKeyCol (default ascending)
-            over(pt.col("tmpLabel")).   // Partition by prefixTempCol
-            alias("rank")
-    )
-	df = df.withColumns(
-        pt.when(pt.col("rank").gt(pt.lit(1))).
-            then(pt.concatStr([pt.lit("A"), pt.col("tmpLabel"), pt.col("rank").cast("String")], { delimiter: "-" })).
-            otherwise(pt.concatStr([pt.lit("A"), pt.col("tmpLabel")], { delimiter: "-" })).
-            alias("seqIdLabel")
-    )
-	df = df.withoutColumns("rank", "tmpLabel")
-
-	//////// add sequence column ////////
-	df = df.addColumns(
-		pt.col(args.sequenceColumnHeader).alias("sequence")
-	)
-	df.save("output.tsv")
-	
-	return ptw.run().getFile("output.tsv")
-}
-
 prepareClonesTsv := func(args) {
 	columns := args.columns
 	datasetSpec := columns.getSpec(args.datasetRef)
@@ -105,62 +40,6 @@ prepareClonesTsv := func(args) {
 	return cloneTable.build()
 }
 
-/**
- * Convert tsv file to fasta file
- * @param fileTsv - tsv file
- * @return fasta file run result
- */
-runTsvToFasta := func(fileTsv) {
-	e := exec.builder().
-		software(prepareFastaSw).
-		mem("8GiB").
-		cpu(1).
-		addFile("input.tsv", fileTsv).
-		arg("-i").arg("input.tsv").
-		arg("-o").arg("output.fasta").
-		arg("--seq_col").arg("sequence").
-		arg("--id_col").arg("seqId").
-		saveFile("output.fasta")
-
-	return e.run()
-}
-
-/**
- * Convert fasta file to tsv file
- * @param fileFasta - fasta file
- * @return tsv file run result
- */
-runFastaToTsv := func(fileFasta) {
-	e := exec.builder().
-		software(fastaToTsvSw).
-		mem("8GiB").
-		cpu(1).
-		addFile("input.fasta", fileFasta).
-		arg("-i").arg("input.fasta").
-		arg("-o").arg("output.tsv").
-		saveFile("output.tsv")
-
-	return e.run()
-}
-
-/**
- * Convert xlsx file to csv file
- * @param fileXlsx - xlsx file
- * @return csv file run result
- */
-runXlsxToCsv := func(fileXlsx) {
-	e := exec.builder().
-		software(xlsxToCsvSw).
-		mem("16GiB").
-		cpu(1).
-		addFile("input.xlsx", fileXlsx).
-		arg("-i").arg("input.xlsx").
-		arg("-o").arg("output.csv").
-		saveFile("output.csv")
-
-	return e.run()
-}
-
 wf.body(func(args) {
 	importFile := file.importFile(args.fileHandle)
 	datasetSpec := args.columns.getSpec(args.datasetRef)
@@ -211,238 +90,102 @@ wf.body(func(args) {
 		xsvType = args.detectedXsvType
 	}
 
-	// Handle XLSX files by converting to CSV first
-	if xsvType == "xlsx" {
-		xlsxToCsvRun := runXlsxToCsv(importFile.file)
-		importFile.file = xlsxToCsvRun.getFile("output.csv")
-		xsvType = "csv"
-	}
-
-	// Handle FASTA files by converting to TSV first
-	if xsvType == "fasta" || xsvType == "fa" {
-		fastaToTsvRun := runFastaToTsv(importFile.file)
-		importFile.file = fastaToTsvRun.getFile("output.tsv")
-		xsvType = "tsv"
-	}
-
-	assayTsv := prepareAssayFile(args, importFile.file, xsvType)
 	clonesTsv := prepareClonesTsv(args)
 
-	// prepare fasta
-	clonesFastaRun := runTsvToFasta(clonesTsv)
-	assayFastaRun := runTsvToFasta(assayTsv)
-	clonesFasta := clonesFastaRun.getFile("output.fasta")
-	assayFasta := assayFastaRun.getFile("output.fasta")
-
-	// Dynamically determine coverage mode by comparing average sequence lengths
-	coverageMode := exec.builder().
-		software(covModeCalcSw).
-		mem("16GiB").
+	// Convert clones TSV to FASTA and check if empty
+	clonesFastaRun := exec.builder().
+		software(prepareFastaSw).
+		mem("8GiB").
 		cpu(1).
-		addFile("clones.fasta", clonesFasta).
-		addFile("assay.fasta", assayFasta).
-		arg("--clones-fasta").arg("clones.fasta").
-		arg("--assay-fasta").arg("assay.fasta").
-		arg("--output").arg("coverage_mode.txt").
-		saveFileContent("coverage_mode.txt").
+		addFile("input.tsv", clonesTsv).
+		arg("-i").arg("input.tsv").
+		arg("-o").arg("output.fasta").
+		arg("--seq_col").arg("sequence").
+		arg("--id_col").arg("seqId").
+		saveFile("output.fasta").
+		saveFileContent("output.fasta").
 		run()
+	clonesFasta := clonesFastaRun.getFile("output.fasta")
+	clonesFastaContent := clonesFastaRun.getFileContent("output.fasta")
 
-	covMode := coverageMode.getFileContent("coverage_mode.txt")
-
-	mmseqsSearchType := "0"
-	if targetSequenceType == "aminoacid" && assaySequenceType == "aminoacid" {
-		//1: amino acid
-		mmseqsSearchType = "1"
-	} else if targetSequenceType == "nucleotide" && assaySequenceType == "nucleotide" {
-		// 3: nucleotide
-		mmseqsSearchType = "3"
-	} else if targetSequenceType == "nucleotide" && assaySequenceType == "aminoacid" {
-		// 4: translated nucleotide alignment
-		mmseqsSearchType = "4"
-	} else if targetSequenceType == "aminoacid" && assaySequenceType == "nucleotide"  {
-		// 2: nucleotide
-		mmseqsSearchType = "2"
-	}
+	// Check if clones are empty before running mmseqs2
+	checkClonesResult := render.create(checkContentEmptyTpl, {
+		content: clonesFastaContent,
+		mode: "raw"
+	})
+	emptyClonesInput := checkClonesResult.output("result")
 
-	runMmseqs := render.create(runAlignmentTpl, {
-		covMode: covMode,
-		mmseqsSearchType: mmseqsSearchType,
+	// Run Analysis Template
+	analysisResult := render.create(analysisTpl, {
+		file: importFile.file,
+		xsvType: xsvType,
+		sequenceColumnHeader: args.sequenceColumnHeader,
+		clonesFasta: clonesFasta,
+		emptyClonesInput: emptyClonesInput,
+		targetSequenceType: targetSequenceType,
+		assaySequenceType: assaySequenceType,
 		coverageThreshold: args.settings.coverageThreshold,
 		identityThreshold: args.settings.identity,
 		similarityType: args.settings.similarityType,
-		clonesFasta: clonesFasta,
-		assayFasta: assayFasta,
 		lessSensitive: args.lessSensitive,
-		mem: args.mem,
-		cpu: args.cpu
+		importColumns: args.importColumns,
+		selectedColumns: args.selectedColumns
+	}, {
+		metaInputs: {
+			mem: args.mem,
+			cpu: args.cpu
+		}
 	})
 
-	mmseqsOutput := runMmseqs.output("mmseqsOutput")
-
-	// @TODO remove header stuff and replace with pt when available (!)
-	addHeaderRunResult := exec.builder().
-		software(addHeaderSw).
-		mem("16GiB").
-		cpu(1).
-		arg("-i").arg("results.tsv").
-		arg("-o").arg("results_with_header.tsv").
-		addFile("results.tsv", mmseqsOutput).
-		saveFile("results_with_header.tsv").
-		saveFileContent("results_with_header.tsv").
-		run()
-
-	mmseqsResultTsv := addHeaderRunResult.getFile("results_with_header.tsv")
-	mmseqsResultTsvContent := addHeaderRunResult.getFileContent("results_with_header.tsv")
-
-	emptyResults := len(text.trim_space(string(mmseqsResultTsvContent))) == 0
+	emptyResults := analysisResult.output("emptyResults")
+	mmseqsOutput := analysisResult.output("mmseqsOutput")
+	
 	blockId := wf.blockId().getDataAsJson()
 
-	assayPframe := undefined
-	epf := undefined
-	assayLinkerPframe := undefined
-	uniqueValuesMap := undefined
-
-	if emptyResults {
-		assayPframe = pframes.emptyPFrame()
-		epf = pframes.emptyPFrame()
-		assayLinkerPframe = pframes.emptyPFrame()
-	} else {
-		//////// Process tables ////////
-		ptw := pt.workflow()
-		df := ptw.frame({
-			file: mmseqsResultTsv,
-			xsvType: "tsv"
-		})
-
-		// Cast columns to ensure correct types for aggregation
-		df = df.withColumns(
-			pt.col("evalue").cast("Float64").alias("evalue"),
-			pt.col("bits").cast("Float64").alias("bits")
-		)
-
-		cols := []
-		for _, col in ["bits", "evalue", "query", "pident", "alnlen", "mismatch", "gapopen", "qstart", "qend", "tstart", "tend"] {
-			cols = append(cols,
-						pt.col(col).maxBy(
-								pt.col("evalue").multiply(-1),
-								pt.col("bits")
-							).alias(col)
-						)
-		}
-		
-		df = df.groupBy("target").agg(cols...)
-		
-		// Add link column for linker pFrame (assayLinkerPframe)
-		df = df.withColumns(
-			pt.lit(1).cast("Int64").alias("link")
-		)
-		
-		df.save("results.tsv")
-
-		// assay data import summary
-		assayDf := ptw.frame({
-			file: assayTsv,
-			xsvType: "tsv"
-		})
-		// import how many matches per assay sequence found
-		assayDf = assayDf.join(
-			df.groupBy("query").agg(
-				pt.col("target").count().alias("queryCount")
-			),
-			{
-				how: "left",
-				leftOn: "seqId",
-				rightOn: "query"
-			}
-		)
-		assayDf.save("assayData.tsv")
-
-		// clones 
-		clonesDf := df.join(assayDf,
-			{
-				how: "left",
-				leftOn: "query",
-				rightOn: "seqId"
-			}
-		)
-
-		clonesDf.save("clonesData.tsv")
-		ptw = ptw.run()
-		
-		//////// Extract unique values from String columns ////////
-		// Find all String columns
-		stringColumns := []
-		for h in args.importColumns {
-			if h.type == "String" && h.header != args.sequenceColumnHeader {
-				stringColumns = append(stringColumns, h.header)
-			}
-		}
-		
-		// Extract unique values for all String columns
-		fileContentsMap := {}
-		if len(stringColumns) > 0 {
-			uniqueValuesWf := pt.workflow().mem("4GiB").cpu(1)
-			baseDf := uniqueValuesWf.frame({
-				file: ptw.getFile("assayData.tsv"),
-				xsvType: "tsv"
-			})
-			// Process each String column to extract unique values
-			for colHeader in stringColumns {
-				uniqueValuesDf := baseDf.select(pt.col(colHeader).alias("value")).groupBy("value").agg(pt.col("value").count().alias("_count"))
-				uniqueValuesDf = uniqueValuesDf.select("value")
-				fileName := "unique_values_" + strings.substituteSpecialCharacters(colHeader) + ".csv"
-				uniqueValuesDf.saveContent(fileName)
-			}
-			// Run once and collect all results
-			uniqueValuesResult := uniqueValuesWf.run()
-			for colHeader in stringColumns {
-				fileName := "unique_values_" + strings.substituteSpecialCharacters(colHeader) + ".csv"
-				fileContentsMap[colHeader] = uniqueValuesResult.getFileContent(fileName)
-			}
-			
-			// Use subtemplate to extract content (getData() only works in subtemplates)
-			extractResult := render.create(extractUniqueValuesTpl, {
-				fileContents: fileContentsMap
-			})
-			uniqueValuesMap = extractResult.output("uniqueValuesMap")
-		}
-		
-		//////// Building outputs & exports ////////
-		buildOutputsResult := render.createEphemeral(buildOutputsTpl, {
-			importColumns: args.importColumns,
-			selectedColumns: args.selectedColumns,
-			sequenceColumnHeader: args.sequenceColumnHeader,
-			sequenceColumnInfo: sequenceColumnInfo,
-			assaySequenceType: assaySequenceType,
-			blockId: blockId,
-			datasetSpec: datasetSpec,
-			bestAlignmentTsv: ptw.getFile("results.tsv"),
-			assayDataTsv: ptw.getFile("assayData.tsv"),
-			clonesDataTsv: ptw.getFile("clonesData.tsv"),
-			uniqueValuesMap: uniqueValuesMap,
-			settings: args.settings,
-			customBlockLabel: args.customBlockLabel,
-			defaultBlockLabel: args.defaultBlockLabel
-		})
-		assayPframe = buildOutputsResult.output("assayPframe")
-		epf = buildOutputsResult.output("epf")
-		assayLinkerPframe = buildOutputsResult.output("assayLinkerPframe")
-	}
+	bestAlignmentTsv := analysisResult.output("bestAlignmentTsv")
+	assayDataTsv := analysisResult.output("assayDataTsv")
+	clonesDataTsv := analysisResult.output("clonesDataTsv")
+	uniqueValuesMap := analysisResult.output("uniqueValuesMap")
+	columnsToImport := analysisResult.output("columnsToImport")
+
+	//////// Build outputs  ////////
+	processOutputsResult := render.create(processOutputsTpl, {
+		emptyClonesInput: emptyClonesInput,
+		assayDataTsv: assayDataTsv,
+		bestAlignmentTsv: bestAlignmentTsv,
+		clonesDataTsv: clonesDataTsv,
+		columnsToImport: columnsToImport,
+		sequenceColumnHeader: args.sequenceColumnHeader,
+		sequenceColumnInfo: sequenceColumnInfo,
+		assaySequenceType: assaySequenceType,
+		datasetSpec: datasetSpec,
+		uniqueValuesMap: uniqueValuesMap,
+		settings: args.settings,
+		customBlockLabel: args.customBlockLabel,
+		defaultBlockLabel: args.defaultBlockLabel,
+		blockId: blockId
+	})
+	assayPframe := processOutputsResult.output("assayPframe")
+	epf := processOutputsResult.output("epf")
+	assayLinkerPframe := processOutputsResult.output("assayLinkerPframe")
 
-	result := {
+	return {
 		outputs: {
 			dataImportHandle: importFile.handle,
 			table: assayPframe,
 			assayLinkerPframe: assayLinkerPframe,
-			mmseqsOutput: mmseqsOutput, // @TODO tmp fix to resolve CID conflicts
-			emptyResults: emptyResults		}
-	}
-
-	if !emptyResults {
-		result.exports = {
+			mmseqsOutput: mmseqsOutput,
+			emptyResults: emptyResults,
+			emptyClonesInput: emptyClonesInput,
+			// Needed for deduplication
+			assayDataTsv: assayDataTsv,
+			bestAlignmentTsv: bestAlignmentTsv,
+			clonesDataTsv: clonesDataTsv,
+			uniqueValuesMap: uniqueValuesMap,
+			columnsToImport: columnsToImport
+		},
+		exports: {
 			epf: epf
 		}
 	}
-
-	return result
 })

package/src/process-outputs.tpl.tengo

@@ -0,0 +1,39 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+render := import("@platforma-sdk/workflow-tengo:render")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+
+buildOutputsTpl := assets.importTemplate(":build-outputs")
+
+self.defineOutputs("assayPframe", "epf", "assayLinkerPframe")
+
+self.body(func(args) {
+	if args.emptyClonesInput {
+		return {
+			assayPframe: {},
+			epf: {},
+			assayLinkerPframe: {}
+		}
+	}
+
+	buildOutputsResult := render.createEphemeral(buildOutputsTpl, {
+		columnsToImport: args.columnsToImport,
+		sequenceColumnHeader: args.sequenceColumnHeader,
+		sequenceColumnInfo: args.sequenceColumnInfo,
+		assaySequenceType: args.assaySequenceType,
+		datasetSpec: args.datasetSpec,
+		bestAlignmentTsv: args.bestAlignmentTsv,
+		assayDataTsv: args.assayDataTsv,
+		clonesDataTsv: args.clonesDataTsv,
+		uniqueValuesMap: args.uniqueValuesMap,
+		settings: args.settings,
+		customBlockLabel: args.customBlockLabel,
+		defaultBlockLabel: args.defaultBlockLabel,
+		blockId: args.blockId
+	})
+
+	return {
+		assayPframe: buildOutputsResult.output("assayPframe"),
+		epf: buildOutputsResult.output("epf"),
+		assayLinkerPframe: buildOutputsResult.output("assayLinkerPframe")
+	}
+})

package/src/run-alignment.tpl.tengo

@@ -18,11 +18,11 @@ self.body(func(args) {
 
 	mem := "8GiB"
 	cpu := 1
-	if !is_undefined(args.mem) {
-		mem = string(args.mem) + "GiB"
+	if !is_undefined(args.metaInputs.mem) {
+		mem = string(args.metaInputs.mem) + "GiB"
 	}
-	if !is_undefined(args.cpu) {
-		cpu = args.cpu
+	if !is_undefined(args.metaInputs.cpu) {
+		cpu = args.metaInputs.cpu
 	}
 
 	mmseqs := exec.builder().