@platforma-open/milaboratories.immune-assay-data.workflow 1.4.2 → 1.5.0

package/.turbo/turbo-build.log

@@ -1,12 +1,16 @@
  WARN  Issue while reading "/home/runner/work/immune-assay-data/immune-assay-data/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.immune-assay-data.workflow@1.4.2 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
+> @platforma-open/milaboratories.immune-assay-data.workflow@1.5.0 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
+Processing "src/build-outputs.tpl.tengo"...
+Processing "src/extract-unique-values.tpl.tengo"...
 Processing "src/main.tpl.tengo"...
 Processing "src/run-alignment.tpl.tengo"...
 No syntax errors found.
   info: Compiling 'dist'...
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/build-outputs.plj.gz
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/extract-unique-values.plj.gz
   info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/run-alignment.plj.gz
   info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/main.plj.gz
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @platforma-open/milaboratories.immune-assay-data.workflow
 
+## 1.5.0
+
+### Minor Changes
+
+- b21c35e: Add isDiscreteFilter and discreteValues annotation to string pcolumn export specs
+
+## 1.4.3
+
+### Patch Changes
+
+- af2e338: Support parquet format (update SDK)
+
 ## 1.4.2
 
 ### Patch Changes

package/dist/index.cjs

@@ -1,4 +1,6 @@
 module.exports = { Templates: {
+  'build-outputs': { type: 'from-file', path: require.resolve('./tengo/tpl/build-outputs.plj.gz') },
+  'extract-unique-values': { type: 'from-file', path: require.resolve('./tengo/tpl/extract-unique-values.plj.gz') },
   'run-alignment': { type: 'from-file', path: require.resolve('./tengo/tpl/run-alignment.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "run-alignment" | "main";
+declare type TplName = "build-outputs" | "extract-unique-values" | "run-alignment" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,5 +1,7 @@
 import { resolve } from 'node:path';
 export const Templates = {
+  'build-outputs': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/build-outputs.plj.gz') },
+  'extract-unique-values': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/extract-unique-values.plj.gz') },
   'run-alignment': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/run-alignment.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "@platforma-open/milaboratories.immune-assay-data.workflow",
-  "version": "1.4.2",
+  "version": "1.5.0",
   "type": "module",
   "description": "Block Workflow",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "^5.3.3",
+    "@platforma-sdk/workflow-tengo": "^5.6.3",
     "@platforma-open/soedinglab.software-mmseqs2": "^1.17.2",
     "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.1.2",
     "@platforma-open/milaboratories.immune-assay-data.add-header": "1.1.2",
-    "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.2",
-    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.1.2"
+    "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "1.1.2",
+    "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "1.1.2"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.3.0",
-    "@platforma-sdk/test": "^1.44.7",
+    "@platforma-sdk/tengo-builder": "^2.3.13",
+    "@platforma-sdk/test": "^1.46.0",
     "vitest": "^2.1.8"
   },
   "scripts": {

package/src/build-outputs.tpl.tengo

@@ -0,0 +1,230 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+strings := import("@platforma-sdk/workflow-tengo:strings")
+
+self.awaitState("uniqueValuesMap", "ResourceReady")
+self.awaitState("datasetSpec", "ResourceReady")
+self.awaitState("assaySequenceType", "ResourceReady")
+self.awaitState("sequenceColumnInfo", "ResourceReady")
+
+assayColumnName := func(header) {
+	return "pl7.app/vdj/assay-data/" + strings.substituteSpecialCharacters(header)
+}
+
+self.body(func(inputs) {
+	//////// Building outputs & exports ////////
+	assayColumns := [
+		{
+			column: "seqIdLabel",
+			spec: {
+				name: "pl7.app/label",
+				valueType: "String",
+				annotations: {
+					"pl7.app/label": "Sequence Id",
+					"pl7.app/table/fontFamily": "monospace"
+				}
+			}
+		},	
+		{
+			column: "queryCount",
+			spec: {
+				name: "pl7.app/vdj/assay/queryCount",
+				valueType: "Int",
+				annotations: {
+					"pl7.app/label": "Matched Clones",
+					"pl7.app/table/orderPriority": "9000"
+				}
+			}
+		},	
+		{
+			column: inputs.sequenceColumnInfo.header,
+			id: strings.substituteSpecialCharacters(inputs.sequenceColumnInfo.header),
+			spec: {
+				name: "pl7.app/vdj/sequence",
+				valueType: "String",
+				domain: {
+					"pl7.app/alphabet": inputs.assaySequenceType
+				},
+				annotations: {
+					"pl7.app/label": inputs.sequenceColumnInfo.header,
+					"pl7.app/table/fontFamily": "monospace",
+					"pl7.app/table/orderPriority": "10000"
+				}
+			}
+		}
+	]
+	
+	columnsToImport := inputs.importColumns
+	if inputs.selectedColumns != undefined && len(inputs.selectedColumns) > 0 {
+		selectedHeaders := {}
+		for header in inputs.selectedColumns {
+			selectedHeaders[header] = true
+		}
+		
+		filteredColumns := []
+		for col in inputs.importColumns {
+			// Always include the main sequence column
+			if col.header == inputs.sequenceColumnHeader || selectedHeaders[col.header] {
+				filteredColumns = append(filteredColumns, col)
+			}
+		}
+		columnsToImport = filteredColumns
+	}
+
+	for h in columnsToImport {
+		if h.header == inputs.sequenceColumnHeader {
+			continue
+		}
+		assayColumns = append(assayColumns, {
+			column: h.header,
+			id: strings.substituteSpecialCharacters(h.header),
+			spec: {
+				name: assayColumnName(h.header),
+				valueType: h.type,
+				annotations: {
+					"pl7.app/label": h.header,
+					"pl7.app/table/orderPriority": "1000"
+				}
+			}
+		})
+	}
+
+	assayImportResults := xsv.importFile(inputs.assayDataTsv, "tsv", {
+		axes: [{
+			column: "seqId",
+			spec: {
+				name: "pl7.app/vdj/assay/sequenceId",
+				type: "String",
+				domain: {
+					"pl7.app/blockId": inputs.blockId
+				},
+				annotations: {
+					"pl7.app/label": "Sequence Id",
+					"pl7.app/table/fontFamily": "monospace"
+				}
+			}
+		}],
+		columns: assayColumns,
+		storageFormat: "Parquet",
+		partitionKeyLength: 0
+	}, {cpu: 1, mem: "16GiB"})
+
+	// "bits", "evalue", "pident"
+	cloneColumns := [
+	{
+		column: "seqIdLabel",
+		spec: {
+			name: "pl7.app/vdj/assay/sequenceIdLabel",
+			valueType: "String",
+			annotations: {
+				"pl7.app/label": "Assay Sequence Id",
+				"pl7.app/table/fontFamily": "monospace",
+				"pl7.app/table/visibility": "optional"
+			}
+		}
+	},
+	{
+		column: "bits",
+		spec: {
+			name: "pl7.app/alignment/bitScore",
+			valueType: "Float",
+			annotations: {
+				"pl7.app/label": "Bit Score",
+				"pl7.app/table/visibility": "optional"
+			}
+		}
+	},
+	{
+		column: "evalue",
+		spec: {
+			name: "pl7.app/alignment/evalue",
+			valueType: "Float",
+			annotations: {
+				"pl7.app/label": "E-value",
+				"pl7.app/table/visibility": "optional"
+			}
+		}
+	},
+	{
+		column: "pident",
+		spec: {
+			name: "pl7.app/alignment/pident",
+			valueType: "Float",
+			annotations: {
+				"pl7.app/label": "Percentage of identical matches",
+				"pl7.app/table/visibility": "optional"
+			}
+		}
+	}]
+
+	for h in columnsToImport {
+		annotations := {
+			"pl7.app/label": h.header,
+			"pl7.app/table/visibility": h.header == inputs.sequenceColumnHeader ? "optional" : "default"
+		}
+		if h.type == "String" && h.header != inputs.sequenceColumnHeader {
+			annotations["pl7.app/isDiscreteFilter"] = "true"
+			annotations["pl7.app/discreteValues"] = inputs.uniqueValuesMap[h.header]
+		}
+		
+		cloneColumns = append(cloneColumns, {
+			column: h.header,
+			id: strings.substituteSpecialCharacters(h.header),
+			spec: {
+				name: assayColumnName(h.header),
+				valueType: h.type,
+				annotations: annotations
+			}
+		})
+	}
+
+	// insert domain
+	for col in cloneColumns {
+		col.spec.domain = maps.deepMerge(col.spec.domain, {
+			"pl7.app/blockId": inputs.blockId
+		})
+	}
+
+	cloneImportResults := xsv.importFile(
+		inputs.clonesDataTsv, "tsv", {
+			axes: [{
+				column: "query",
+				spec: inputs.datasetSpec.axesSpec[1]
+			}],
+			columns: cloneColumns,
+			storageFormat: "Parquet",
+			partitionKeyLength: 0
+		},
+		{ splitDataAndSpec: true, cpu: 1, mem: "16GiB" }
+	)
+
+	// Create informative label with relevant matching parameters
+	identityStr := string(inputs.settings.identity)
+	coverageStr := string(inputs.settings.coverageThreshold)
+	similarityTypeStr := inputs.settings.similarityType == "sequence-identity" ? "Exact Match" : "BLOSUM"
+	
+	traceLabel := "Assay Data (sim:" + similarityTypeStr + ", ident:" + identityStr + ", cov:" + coverageStr + ")"
+	
+	trace := pSpec.makeTrace(inputs.datasetSpec,
+		{
+			type: "milaboratories.immune-assay-data",
+			importance: 40,
+			label: traceLabel
+		})
+
+	epfB := pframes.pFrameBuilder()
+	for k, v in cloneImportResults {
+		epfB.add(k, trace.inject(v.spec), v.data)
+	}
+	epf := epfB.build()
+	assayPframe := pframes.exportFrame(assayImportResults)
+
+	return {
+		assayPframe: assayPframe,
+		epf: epf
+	}
+})
+

package/src/extract-unique-values.tpl.tengo

@@ -0,0 +1,32 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+json := import("json")
+text := import("text")
+
+self.defineOutputs("uniqueValuesMap")
+
+self.body(func(inputs) {
+	fileContents := inputs.fileContents
+	uniqueValuesMap := {}
+
+	for header, contentField in fileContents {
+		// In subtemplate, we can call getData() directly on exec fileContent fields
+		// (following clonotype-enrichment pattern)
+		contentBytes := contentField.getData()
+		content := string(contentBytes)
+		lines := text.split(text.trim_space(content), "\n")
+		
+		if len(lines) > 1 {
+			// Skip header and collect values
+			values := lines[1:]
+			// JSON encode the array of strings and convert to string
+			encodedBytes := json.encode(values)
+			uniqueValuesMap[header] = string(encodedBytes)
+		}
+	}
+
+	return {
+		uniqueValuesMap: uniqueValuesMap
+	}
+})
+

package/src/main.tpl.tengo

@@ -14,6 +14,8 @@ text := import("text")
 render := import("@platforma-sdk/workflow-tengo:render")
 strings := import("@platforma-sdk/workflow-tengo:strings")
 runAlignmentTpl := assets.importTemplate(":run-alignment")
+extractUniqueValuesTpl := assets.importTemplate(":extract-unique-values")
+buildOutputsTpl := assets.importTemplate(":build-outputs")
 
 prepareFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.prepare-fasta:main")
 fastaToTsvSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv:main")
@@ -140,10 +142,6 @@ runFastaToTsv := func(fileFasta) {
 	return e.run()
 }
 
-assayColumnName := func(header) {
-	return "pl7.app/vdj/assay-data/" + strings.substituteSpecialCharacters(header)
-}
-
 wf.body(func(args) {
 	importFile := file.importFile(args.fileHandle)
 	datasetSpec := args.columns.getSpec(args.datasetRef)
@@ -256,6 +254,7 @@ wf.body(func(args) {
 		arg("-o").arg("results_with_header.tsv").
 		addFile("results.tsv", mmseqsOutput).
 		saveFile("results_with_header.tsv").
+		saveFileContent("results_with_header.tsv").
 		run()
 
 	mmseqsResultTsv := addHeaderRunResult.getFile("results_with_header.tsv")
@@ -266,6 +265,7 @@ wf.body(func(args) {
 
 	assayPframe := undefined
 	epf := undefined
+	uniqueValuesMap := undefined
 
 	if emptyResults {
 		assayPframe = pframes.emptyPFrame()
@@ -327,205 +327,60 @@ wf.body(func(args) {
 		clonesDf.save("clonesData.tsv")
 		ptw = ptw.run()
 		
-		//////// Building outputs & exports ////////
-		assayColumns := [
-			{
-				column: "seqIdLabel",
-				spec: {
-					name: "pl7.app/label",
-					valueType: "String",
-					annotations: {
-						"pl7.app/label": "Sequence Id",
-						"pl7.app/table/fontFamily": "monospace"
-					}
-				}
-			},	
-			{
-				column: "queryCount",
-				spec: {
-					name: "pl7.app/vdj/assay/queryCount",
-					valueType: "Int",
-					annotations: {
-						"pl7.app/label": "Matched Clones",
-						"pl7.app/table/orderPriority": "9000"
-					}
-				}
-			},	
-			{
-				column: sequenceColumnInfo.header,
-				id: strings.substituteSpecialCharacters(sequenceColumnInfo.header),
-				spec: {
-					name: "pl7.app/vdj/sequence",
-					valueType: "String",
-					domain: {
-						"pl7.app/alphabet": assaySequenceType
-					},
-					annotations: {
-						"pl7.app/label": sequenceColumnInfo.header,
-						"pl7.app/table/fontFamily": "monospace",
-						"pl7.app/table/orderPriority": "10000"
-					}
-				}
-			}
-		]
-		
-		columnsToImport := args.importColumns
-		if args.selectedColumns != undefined && len(args.selectedColumns) > 0 {
-			selectedHeaders := {}
-			for header in args.selectedColumns {
-				selectedHeaders[header] = true
+		//////// Extract unique values from String columns ////////
+		// Find all String columns
+		stringColumns := []
+		for h in args.importColumns {
+			if h.type == "String" && h.header != args.sequenceColumnHeader {
+				stringColumns = append(stringColumns, h.header)
 			}
-			
-			filteredColumns := []
-			for col in args.importColumns {
-				// Always include the main sequence column
-				if col.header == args.sequenceColumnHeader || selectedHeaders[col.header] {
-					filteredColumns = append(filteredColumns, col)
-				}
-			}
-			columnsToImport = filteredColumns
 		}
-
-		for h in columnsToImport {
-			if h.header == args.sequenceColumnHeader {
-				continue
-			}
-			assayColumns = append(assayColumns, {
-				column: h.header,
-				id: strings.substituteSpecialCharacters(h.header),
-				spec: {
-					name: assayColumnName(h.header),
-					valueType: h.type,
-					annotations: {	
-						"pl7.app/label": h.header,
-						"pl7.app/table/orderPriority": "1000"
-					}
-				}
+		
+		// Extract unique values for all String columns
+		fileContentsMap := {}
+		if len(stringColumns) > 0 {
+			uniqueValuesWf := pt.workflow().mem("4GiB").cpu(1)
+			baseDf := uniqueValuesWf.frame({
+				file: ptw.getFile("assayData.tsv"),
+				xsvType: "tsv"
 			})
-		}
-
-		assayImportResults := xsv.importFile(ptw.getFile("assayData.tsv"), "tsv", {
-			axes: [{
-				column: "seqId",
-				spec: {
-					name: "pl7.app/vdj/assay/sequenceId",
-					type: "String",
-					domain: {
-						"pl7.app/blockId": blockId
-					},
-					annotations: {
-						"pl7.app/label": "Sequence Id",
-						"pl7.app/table/fontFamily": "monospace"
-					}
-				}
-			}],
-			columns: assayColumns,
-			annotations: {
-				"pl7.app/isAnchor": "true"
-			}
-		}, {cpu: 1, mem: "16GiB"})
-
-		// "bits", "evalue", "pident"
-		cloneColumns := [
-		{
-			column: "seqIdLabel",
-			spec: {
-				name: "pl7.app/vdj/assay/sequenceIdLabel",
-				valueType: "String",
-				annotations: {
-					"pl7.app/label": "Assay Sequence Id",
-					"pl7.app/table/fontFamily": "monospace",
-					"pl7.app/table/visibility": "optional"
-				}
-			}
-		},
-		{
-			column: "bits",
-			spec: {
-				name: "pl7.app/alignment/bitScore",
-				valueType: "Float",
-				annotations: {
-					"pl7.app/label": "Bit Score",
-					"pl7.app/table/visibility": "optional"
-				}
+			// Process each String column to extract unique values
+			for colHeader in stringColumns {
+				uniqueValuesDf := baseDf.select(pt.col(colHeader).alias("value")).groupBy("value").agg(pt.col("value").count().alias("_count"))
+				uniqueValuesDf = uniqueValuesDf.select("value")
+				fileName := "unique_values_" + strings.substituteSpecialCharacters(colHeader) + ".csv"
+				uniqueValuesDf.saveContent(fileName)
 			}
-		},
-		{
-			column: "evalue",
-			spec: {
-				name: "pl7.app/alignment/evalue",
-				valueType: "Float",
-				annotations: {
-					"pl7.app/label": "E-value",
-					"pl7.app/table/visibility": "optional"
-				}
+			// Run once and collect all results
+			uniqueValuesResult := uniqueValuesWf.run()
+			for colHeader in stringColumns {
+				fileName := "unique_values_" + strings.substituteSpecialCharacters(colHeader) + ".csv"
+				fileContentsMap[colHeader] = uniqueValuesResult.getFileContent(fileName)
 			}
-		},
-		{
-			column: "pident",
-			spec: {
-				name: "pl7.app/alignment/pident",
-				valueType: "Float",
-				annotations: {
-					"pl7.app/label": "Percentage of identical matches",
-					"pl7.app/table/visibility": "optional"
-				}
-			}
-		}]
-
-		for h in columnsToImport {
-			cloneColumns = append(cloneColumns, {
-				column: h.header,
-				id: strings.substituteSpecialCharacters(h.header),
-				spec: {
-					name: assayColumnName(h.header),
-					valueType: h.type,
-					annotations: {	
-						"pl7.app/label": h.header,
-						"pl7.app/table/visibility": h.header == args.sequenceColumnHeader ? "optional" : "default"
-					}
-				}
-			})
-		}
-
-		// insert domain
-		for col in cloneColumns {
-			col.spec.domain = maps.deepMerge(col.spec.domain, {
-				"pl7.app/blockId": blockId
+			
+			// Use subtemplate to extract content (getData() only works in subtemplates)
+			extractResult := render.create(extractUniqueValuesTpl, {
+				fileContents: fileContentsMap
 			})
+			uniqueValuesMap = extractResult.output("uniqueValuesMap")
 		}
-
-		cloneImportResults := xsv.importFile(
-			ptw.getFile("clonesData.tsv"), "tsv", {
-				axes: [{
-					column: "query",
-					spec: datasetSpec.axesSpec[1]
-				}],
-				columns: cloneColumns
-			},
-			{ splitDataAndSpec: true, cpu: 1, mem: "16GiB" }
-		)
-
-		// Create informative label with relevant matching parameters
-		identityStr := string(args.settings.identity)
-		coverageStr := string(args.settings.coverageThreshold)
-		similarityTypeStr := args.settings.similarityType == "sequence-identity" ? "Exact Match" : "BLOSUM"
 		
-		traceLabel := "Assay Data (sim:" + similarityTypeStr + ", ident:" + identityStr + ", cov:" + coverageStr + ")"
-		
-		trace := pSpec.makeTrace(datasetSpec,
-			{
-				type: "milaboratories.immune-assay-data",
-				importance: 40,
-				label: traceLabel
-			})
-
-		epfB := pframes.pFrameBuilder()
-		for k, v in cloneImportResults {
-			epfB.add(k, trace.inject(v.spec), v.data)
-		}
-		epf = epfB.build()
-		assayPframe = pframes.exportFrame(assayImportResults)
+		//////// Building outputs & exports ////////
+		buildOutputsResult := render.createEphemeral(buildOutputsTpl, {
+			importColumns: args.importColumns,
+			selectedColumns: args.selectedColumns,
+			sequenceColumnHeader: args.sequenceColumnHeader,
+			sequenceColumnInfo: sequenceColumnInfo,
+			assaySequenceType: assaySequenceType,
+			blockId: blockId,
+			datasetSpec: datasetSpec,
+			assayDataTsv: ptw.getFile("assayData.tsv"),
+			clonesDataTsv: ptw.getFile("clonesData.tsv"),
+			uniqueValuesMap: uniqueValuesMap,
+			settings: args.settings
+		})
+		assayPframe = buildOutputsResult.output("assayPframe")
+		epf = buildOutputsResult.output("epf")
 	}
 
 	result := {
@@ -533,8 +388,7 @@ wf.body(func(args) {
 			dataImportHandle: importFile.handle,
 			table: assayPframe,
 			mmseqsOutput: mmseqsOutput, // @TODO tmp fix to resolve CID conflicts
-			emptyResults: emptyResults
-		}
+			emptyResults: emptyResults		}
 	}
 
 	if !emptyResults {

package/src/run-alignment.tpl.tengo

@@ -20,7 +20,6 @@ self.body(func(args) {
 		software(mmseqsSw).
 		mem("32GiB").
 		cpu(1).
-		dontSaveStdoutOrStderr(). // important to avoid CID conflict problems coming from different stdout output on same datasets
 		arg("easy-search").
 		arg("clones.fasta").
 		arg("assay.fasta").