@platforma-open/milaboratories.immune-assay-data.workflow 1.0.1

package/.turbo/turbo-build.log

@@ -0,0 +1,11 @@
+ WARN  Issue while reading "/home/runner/work/immune-assay-data/immune-assay-data/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
+
+> @platforma-open/milaboratories.immune-assay-data.workflow@1.0.1 build /home/runner/work/immune-assay-data/immune-assay-data/workflow
+> rm -rf dist && pl-tengo check && pl-tengo build
+
+Processing "src/main.tpl.tengo"...
+No syntax errors found.
+  info: Compiling 'dist'...
+  info:   - writing /home/runner/work/immune-assay-data/immune-assay-data/workflow/dist/tengo/tpl/main.plj.gz
+  info: Template Pack build done.
+  info: Template Pack build done.

package/CHANGELOG.md

@@ -0,0 +1,9 @@
+# @platforma-open/milaboratories.immune-assay-data.workflow
+
+## 1.0.1
+
+### Patch Changes
+
+- f59e17e: Initial release
+- Updated dependencies [f59e17e]
+  - @platforma-open/milaboratories.immune-assay-data.prepare-fasta@1.0.2

package/dist/index.cjs

@@ -0,0 +1,3 @@
+module.exports = { Templates: {
+  'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
+}};

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
+declare type TplName = "main";
+declare const Templates: Record<TplName, TemplateFromFile>;
+export { Templates };

package/dist/index.js

@@ -0,0 +1,4 @@
+import { resolve } from 'node:path';
+export const Templates = {
+  'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
+};

package/format.el

@@ -0,0 +1,43 @@
+;; This program formats all files inside src directory. Usage: emacs --script ./format.el
+
+(defun install-go-mode ()
+  "Installs go-mode"
+  (require 'package)
+  (add-to-list 'package-archives
+               '("melpa-stable" . "https://stable.melpa.org/packages/"))
+  (package-initialize)
+  (unless package-archive-contents
+    (package-refresh-contents))
+
+  (package-install 'go-mode t)
+  (require 'go-mode))
+
+;; spaces -> tabs only at the beginning of lines
+(setq tabify-regexp "^\t* [ \t]+")
+
+(defun format-file (file)
+  "Formats a file according to slightly changed Go rules"
+  (message "Format %s" file)
+  (save-excursion
+    (find-file file)
+    (delete-trailing-whitespace)            ;; deletes whitespaces
+    (go-mode)                               ;; sets golang rules for indentation
+    (tabify (point-min) (point-max))        ;; spaces -> tabs in the whole file
+    (indent-region (point-min) (point-max)) ;; indentation in the whole file
+    (save-buffer)))                         ;; save file
+
+(install-go-mode)
+
+;; change syntax of a standard go-mode a bit
+(advice-add
+ 'go--in-composite-literal-p
+ :filter-return
+ (lambda (&rest r) t))
+
+;; find all files in src
+(setq files (directory-files-recursively "src" "\\.tengo\\'"))
+
+;; call format on every file.
+(dolist (file files)
+  (format-file file))
+

package/index.d.ts

@@ -0,0 +1,4 @@
+declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
+declare type TplName = "main";
+declare const Templates: Record<TplName, TemplateFromFile>;
+export { Templates };

package/index.js

@@ -0,0 +1,3 @@
+module.exports = { Templates: {
+  'main': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/main.plj.gz') }
+}}

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@platforma-open/milaboratories.immune-assay-data.workflow",
+  "version": "1.0.1",
+  "type": "module",
+  "description": "Block Workflow",
+  "dependencies": {
+    "@platforma-sdk/workflow-tengo": "^4.6.1",
+    "@platforma-open/soedinglab.software-mmseqs2": "^1.0.0",
+    "@platforma-open/milaboratories.immune-assay-data.prepare-fasta": "1.0.2",
+    "@platforma-open/milaboratories.immune-assay-data.add-header": "1.0.1"
+  },
+  "devDependencies": {
+    "@platforma-sdk/tengo-builder": "^2.1.5",
+    "@platforma-sdk/test": "^1.31.16",
+    "vitest": "^2.1.8"
+  },
+  "scripts": {
+    "build": "rm -rf dist && pl-tengo check && pl-tengo build",
+    "test": "vitest",
+    "format": "/usr/bin/env emacs --script ./format.el"
+  }
+}

package/src/main.tpl.tengo

@@ -0,0 +1,408 @@
+// light block with no workflow
+wf := import("@platforma-sdk/workflow-tengo:workflow")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+file := import("@platforma-sdk/workflow-tengo:file")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+assets:= import("@platforma-sdk/workflow-tengo:assets")
+maps:= import("@platforma-sdk/workflow-tengo:maps")
+xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+
+prepareFastaSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.prepare-fasta:main")
+mmseqsSw := assets.importSoftware("@platforma-open/soedinglab.software-mmseqs2:main")
+addHeaderSw := assets.importSoftware("@platforma-open/milaboratories.immune-assay-data.add-header:main")
+
+wf.prepare(func(args){
+	bundleBuilder := wf.createPBundleBuilder()
+	bundleBuilder.addAnchor("main", args.datasetRef) 
+	bundleBuilder.addSingle(args.targetRef)
+	return {
+		columns: bundleBuilder.build()
+	}
+})
+
+prepareAssayTsv := func(args, file) {
+	// assign ids to assay sequences
+	ptw := pt.workflow()
+	df := ptw.frame({
+		file: file,
+		xsvType: "tsv" // @TODO (!!!)
+	})
+
+	//////// calculate sequence id ////////
+	df = df.addColumns(
+		pt.col(args.sequenceColumnHeader).hash("sha256", "base64_alphanumeric", 120).alias("seqId")
+	)
+	//////// add label to ids ////////
+	df = df.withColumns(
+        pt.col("seqId").
+            strReplace("\\d", "", { replaceAll: true }).
+            strSlice(0, 5).               // Take first 5 characters
+            strToUpper().                 // Convert to uppercase
+            alias("tmpLabel")
+    )
+	df = df.withColumns(
+        pt.rank(pt.col("seqId")).  // Rank based on clonotypeKeyCol (default ascending)
+            over(pt.col("tmpLabel")).   // Partition by prefixTempCol
+            alias("rank")
+    )
+	df = df.withColumns(
+        pt.when(pt.col("rank").gt(pt.lit(1))).
+            then(pt.concatStr([pt.lit("A"), pt.col("tmpLabel"), pt.col("rank").cast("String")], { delimiter: "-" })).
+            otherwise(pt.concatStr([pt.lit("A"), pt.col("tmpLabel")], { delimiter: "-" })).
+            alias("seqIdLabel")
+    )
+	df = df.withoutColumns("rank", "tmpLabel")
+
+	//////// add sequence column ////////
+	df = df.addColumns(
+		pt.col(args.sequenceColumnHeader).alias("sequence")
+	)
+	df.save("output.tsv")
+	
+	return ptw.run().getFile("output.tsv")
+}
+
+prepareClonesTsv := func(args) {
+	columns := args.columns
+	datasetSpec := columns.getSpec(args.datasetRef)
+
+	cloneTable := pframes.tsvFileBuilder()
+	
+	cloneTable.setAxisHeader(datasetSpec.axesSpec[1].name, "seqId")
+	cloneTable.add(columns.getColumn(args.targetRef), {header: "sequence"})
+
+	return cloneTable.build()
+}
+
+/**
+ * Convert tsv file to fasta file
+ * @param fileTsv - tsv file
+ * @return fasta file
+ */
+tsv2Fasta := func(fileTsv) {
+	e := exec.builder().
+		software(prepareFastaSw).
+		addFile("input.tsv", fileTsv).
+		arg("-i").arg("input.tsv").
+		arg("-o").arg("output.fasta").
+		arg("--seq_col").arg("sequence").
+		arg("--id_col").arg("seqId").
+		saveFile("output.fasta")
+
+	return e.run().getFile("output.fasta")
+}
+
+wf.body(func(args) {
+	importFile := file.importFile(args.fileHandle)
+	datasetSpec := args.columns.getSpec(args.datasetRef)
+	targetSpec := args.columns.getSpec(args.targetRef)
+	
+	// aminoacid or nucleotide
+	sequenceColumnInfo := undefined
+	for col in args.importColumns {
+		if col.header == args.sequenceColumnHeader {
+			sequenceColumnInfo = col
+			break
+		}
+	}
+
+	// aminoacid or nucleotide
+	targetSequenceType := targetSpec.domain["pl7.app/alphabet"]
+	assaySequenceType := sequenceColumnInfo.sequenceType
+
+	if targetSequenceType == undefined {
+		ll.panic("Target sequence type is undefined")
+	}
+	
+	if assaySequenceType == undefined {
+		ll.panic("Assay sequence type is undefined")
+	}
+
+	assayTsv := prepareAssayTsv(args, importFile.file)
+	clonesTsv := prepareClonesTsv(args)
+
+	// prepare fasta
+	clonesFasta := tsv2Fasta(clonesTsv)
+	assayFasta := tsv2Fasta(assayTsv)
+	
+	mmseqsSearchType := "0"
+	if targetSequenceType == "aminoacid" && assaySequenceType == "aminoacid" {
+		//1: amino acid
+		mmseqsSearchType = "1"
+	} else if targetSequenceType == "nucleotide" && assaySequenceType == "nucleotide" {
+		// 3: nucleotide
+		mmseqsSearchType = "3"
+	} else if targetSequenceType == "nucleotide" && assaySequenceType == "aminoacid" {
+		// 4: translated nucleotide alignment
+		mmseqsSearchType = "4"
+	} else if targetSequenceType == "aminoacid" && assaySequenceType == "nucleotide"  {
+		// 2: nucleotide
+		mmseqsSearchType = "2"
+	}
+	// run search
+	mmseqs := exec.builder().
+		software(mmseqsSw).
+		dontSaveStdoutOrStderr(). // important to avoid CID conflict problems coming from different stdout output on same datasets
+		arg("easy-search").
+		arg("clones.fasta").
+		arg("assay.fasta").
+		arg("results.tsv").
+		arg("tmp").
+		arg("--search-type").arg(mmseqsSearchType).
+		arg("--cov-mode").arg(string(args.settings.coverageMode)).
+		arg("-c").arg(string(args.settings.coverageThreshold)).
+		addFile("clones.fasta", clonesFasta).
+		addFile("assay.fasta", assayFasta).
+		saveFile("results.tsv").
+		run()
+
+	mmseqsOutput := mmseqs.getFile("results.tsv")
+	// @TODO remove header stuff and replace with pt when available (!)
+	mmseqsResultTsv := exec.builder().
+		software(addHeaderSw).
+		arg("-i").arg("results.tsv").
+		arg("-o").arg("results_with_header.tsv").
+		addFile("results.tsv", mmseqsOutput).
+		saveFile("results_with_header.tsv").
+		run().
+		getFile("results_with_header.tsv")
+
+
+	//////// Process tables ////////
+
+	ptw := pt.workflow()
+	df := ptw.frame({
+		file: mmseqsResultTsv,
+		xsvType: "tsv"
+	})
+
+	cols := []
+	for _, col in ["bits", "evalue", "target", "pident", "alnlen", "mismatch", "gapopen", "qstart", "qend", "tstart", "tend"] {
+		cols = append(cols,
+					  pt.col(col).maxBy(
+							pt.col("evalue").multiply(-1),
+							pt.col("bits")
+					  	).alias(col)
+					  )
+	}
+	
+	df = df.groupBy("query").agg(cols...)
+	df.save("results.tsv")
+
+	// assay data import summary
+	assayDf := ptw.frame({
+		file: assayTsv,
+		xsvType: "tsv"
+	})
+	// import how many matches per assay sequence found
+	assayDf = assayDf.join(
+		df.groupBy("target").agg(
+			pt.col("query").count().alias("queryCount")
+		),
+		{
+			how: "left",
+			leftOn: "seqId",
+			rightOn: "target"
+		}
+	)
+	assayDf.save("assayData.tsv")
+
+	// clones 
+	clonesDf := df.join(assayDf,
+		{
+			how: "left",
+			leftOn: "target",
+			rightOn: "seqId"
+		}
+	)
+
+	clonesDf.save("clonesData.tsv")
+	ptw = ptw.run()
+	
+	//////// Building outputs & exports ////////
+	blockId := wf.blockId().getDataAsJson()
+
+	assayColumns := [
+		{
+			column: "seqIdLabel",
+			spec: {
+				name: "pl7.app/label",
+				valueType: "String",
+				annotations: {
+					"pl7.app/label": "Sequence Id",
+					"pl7.app/table/fontFamily": "monospace"
+				}
+			}
+		},	
+		{
+			column: "queryCount",
+			spec: {
+				name: "pl7.app/vdj/assay/queryCount",
+				valueType: "Int",
+				annotations: {
+					"pl7.app/label": "Matched Clones",
+					"pl7.app/table/orderPriority": "9000"
+				}
+			}
+		},	
+		{
+			column: sequenceColumnInfo.header,
+			spec: {
+				name: "pl7.app/vdj/sequence",
+				valueType: "String",
+				domain: {
+					"pl7.app/alphabet": assaySequenceType
+				},
+				annotations: {
+					"pl7.app/label": sequenceColumnInfo.header,
+					"pl7.app/table/fontFamily": "monospace",
+					"pl7.app/table/orderPriority": "10000"
+				}
+			}
+		}
+	]
+	for h in args.importColumns {
+		if h.header == args.sequenceColumnHeader {
+			continue
+		}
+		assayColumns = append(assayColumns, {
+			column: h.header,
+			spec: {
+				name: h.header,
+				valueType: h.type,
+				annotations: {	
+					"pl7.app/label": h.header,
+					"pl7.app/table/orderPriority": "1000"
+				}
+			}
+		})
+	}
+
+	assayImportResults := xsv.importFile(ptw.getFile("assayData.tsv"), "tsv", {
+		axes: [{
+			column: "seqId",
+			spec: {
+				name: "pl7.app/vdj/assay/sequenceId",
+				type: "String",
+				domain: {
+					"pl7.app/blockId": blockId
+				},
+				annotations: {
+					"pl7.app/label": "Sequence Id",
+					"pl7.app/table/fontFamily": "monospace"
+				}
+			}
+		}],
+		columns: assayColumns,
+		annotations: {
+			"pl7.app/isAnchor": "true"
+		}
+	})
+
+    // "bits", "evalue", "pident"
+	cloneColumns := [
+	{
+		column: "target",
+		spec: {
+			name: "pl7.app/vdj/assay/sequenceId",
+			valueType: "String",
+			annotations: {
+				"pl7.app/label": "Assay Sequence Id",
+				"pl7.app/table/defaultVisibility": "optional"
+			}
+		}
+	},
+	{
+		column: "bits",
+		spec: {
+			name: "pl7.app/alignment/bitScore",
+			valueType: "Float",
+			annotations: {
+				"pl7.app/label": "Bit Score",
+				"pl7.app/table/defaultVisibility": "optional"
+			}
+		}
+	},
+	{
+		column: "evalue",
+		spec: {
+			name: "pl7.app/alignment/evalue",
+			valueType: "Float",
+			annotations: {
+				"pl7.app/label": "E-value",
+				"pl7.app/table/defaultVisibility": "optional"
+			}
+		}
+	},
+	{
+		column: "pident",
+		spec: {
+			name: "pl7.app/alignment/pident",
+			valueType: "Float",
+			annotations: {
+				"pl7.app/label": "Percentage of identical matches",
+				"pl7.app/table/defaultVisibility": "optional"
+			}
+		}
+	}]
+
+	for h in args.importColumns {
+		cloneColumns = append(cloneColumns, {
+			column: h.header,
+			spec: {
+				name: h.header,
+				valueType: h.type,
+				annotations: {	
+					"pl7.app/label": h.header,
+					"pl7.app/table/defaultVisibility": h.header == args.sequenceColumnHeader ? "optional" : "default"
+				}
+			}
+		})
+	}
+
+	// insert domain
+	for col in cloneColumns {
+		col.spec.domain = maps.deepMerge(col.spec.domain, {
+			"pl7.app/blockId": blockId
+		})
+	}
+
+    cloneImportResults := xsv.importFile(
+		ptw.getFile("clonesData.tsv"), "tsv", {
+			axes: [{
+				column: "query",
+				spec: datasetSpec.axesSpec[1]
+			}],
+			columns: cloneColumns
+		},
+		{ splitDataAndSpec: true }
+	)
+
+	trace := pSpec.makeTrace(datasetSpec,
+        {
+            type: "milaboratories.immune-assay-data",
+            importance: 30,
+            label: "Assay Data"
+        })
+
+	epf := pframes.pFrameBuilder()
+	for k, v in cloneImportResults {
+		epf.add(k, trace.inject(v.spec), v.data)
+	}
+	epf = epf.build()
+	
+	return {
+		outputs: {
+			dataImportHandle: importFile.handle,
+			table: pframes.exportFrame(assayImportResults),
+			mmseqsOutput: mmseqsOutput // @TODO tmp fix to resolve CID conflicts
+		},
+		exports: {
+			epf: epf
+		}
+	}
+})

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "strict": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "sourceMap": true,
+    "declaration": true
+  },
+  "types": [],
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/vitest.config.mts

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    watch: false,
+    maxConcurrency: 3,
+    testTimeout: 5000
+  }
+});