@platforma-open/milaboratories.mixcr-shm-trees.workflow 2.3.3 → 3.0.0

package/.turbo/turbo-build.log

@@ -0,0 +1,22 @@
+
+> @platforma-open/milaboratories.mixcr-shm-trees.workflow@3.0.0 build /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow
+> rm -rf dist && pl-tengo check && pl-tengo build
+
+Processing "src/export-settings.lib.tengo"...
+Processing "src/main.tpl.tengo"...
+Processing "src/prepare-donor-column.lib.tengo"...
+Processing "src/process.tpl.tengo"...
+Processing "src/reconstruct-shm-trees.tpl.tengo"...
+Processing "src/soi-export.lib.tengo"...
+Processing "src/soi.tpl.tengo"...
+No syntax errors found.
+  info: Compiling 'dist'...
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/lib/export-settings.lib.tengo
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/lib/prepare-donor-column.lib.tengo
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/lib/soi-export.lib.tengo
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/reconstruct-shm-trees.plj.gz
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/soi.plj.gz
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/process.plj.gz
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/main.plj.gz
+  info: 
+  info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @platforma-open/milaboratories.mixcr-shm-trees.workflow
 
+## 3.0.0
+
+### Major Changes
+
+- 84fec04: Sequence Search Support
+
+## 2.3.4
+
+### Patch Changes
+
+- ef7df65: SDK upgrade
+
 ## 2.3.3
 
 ### Patch Changes

package/dist/index.cjs

@@ -1,5 +1,6 @@
 module.exports = { Templates: {
   'reconstruct-shm-trees': { type: 'from-file', path: require.resolve('./tengo/tpl/reconstruct-shm-trees.plj.gz') },
+  'soi': { type: 'from-file', path: require.resolve('./tengo/tpl/soi.plj.gz') },
   'process': { type: 'from-file', path: require.resolve('./tengo/tpl/process.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "reconstruct-shm-trees" | "process" | "main";
+declare type TplName = "reconstruct-shm-trees" | "soi" | "process" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,6 +1,7 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'reconstruct-shm-trees': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/reconstruct-shm-trees.plj.gz') },
+  'soi': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/soi.plj.gz') },
   'process': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/process.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/dist/tengo/lib/export-settings.lib.tengo

@@ -444,7 +444,7 @@ shmTreeNodesTableOptions := func(dataDescription) {
 // export data that is unique for clones, but not unique for a node
 // (different clones could be in the same topology node, for example, different time points)
 shmTreeNodesWithClonesTableOptions := func(dataDescription, donorColumn) {
-    donorColumnSpec := donorColumn.get("spec").getDataAsJson()
+    donorColumnSpec := donorColumn.spec
 
     axes := []
     columns := []

package/dist/tengo/lib/prepare-donor-column.lib.tengo

@@ -20,7 +20,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
 	//        axes[0]: sampleId
 	//          value: fileRef resource
 
-    donorColumnSpec := donorColumn.get("spec").getDataAsJson()
+    donorColumnSpec := donorColumn.spec
 
     sampleIdAxis := donorColumnSpec.axesSpec[0]
 
@@ -53,7 +53,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
     sampleToDonor := {}
 
     // columns with meta could be fetched as data direcctly
-	for k, v in donorColumn.get("data").getDataAsJson()["data"] {
+	for k, v in donorColumn.data.getDataAsJson()["data"] {
         sampleId := json.decode(k)[0]
         sampleToDonor[sampleId] = v
 	}
@@ -63,7 +63,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
 
     // collect all the clns files that we have into pColumn
 	for clonotypingBlockId, dataset in datasets {
-		for sKey, fileRef in dataset.get("data").inputs() {
+		for sKey, fileRef in dataset.data.inputs() {
             sampleId := json.decode(sKey)[0]
             donor := sampleToDonor[sampleId]
             dataBuilder.createInputField(json.encode([donor, sampleId, clonotypingBlockId])).set(fileRef)

package/dist/tengo/lib/soi-export.lib.tengo

@@ -0,0 +1,39 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+soiResultImportColumns := func(dbParameters) {
+	if dbParameters.searchParameters.type == "tree_search_top" {
+		return [ {
+			"column": "topHit",
+			"id": "topHit",
+			"spec": {
+				"name": "pl7.app/search/topHit",
+				"valueType": "String",
+				"annotations": {
+					"pl7.app/label": "Top Hit " + dbParameters.name
+				}
+			},
+			"domain": {
+				"pl7.app/list": dbParameters.id
+			}
+		}, {
+			"column": "mutations",
+			"id": "mutations",
+			"spec": {
+				"name": "pl7.app/search/numberOfMutations",
+				"valueType": "Int",
+				"annotations": {
+					"pl7.app/label": "Number of mutations " + dbParameters.name
+				}
+			},
+			"domain": {
+				"pl7.app/list": dbParameters.id
+			}
+		} ]
+	} else {
+		ll.panic("Unknown search mode: " + dbParameters.searchParameters.type)
+	}
+}
+
+export ll.toStrict({
+	soiResultImportColumns: soiResultImportColumns
+})

package/package.json

@@ -1,18 +1,20 @@
 {
   "name": "@platforma-open/milaboratories.mixcr-shm-trees.workflow",
-  "version": "2.3.3",
+  "version": "3.0.0",
   "type": "module",
   "description": "Tengo-based template",
   "//": {
     "build": "node ./scripts/build-static.mjs src/pfconv_params.json src/pfconv_params.lib.tengo && rm -rf dist && pl-tengo check && pl-tengo build && ./create_tags.sh"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^1.17.3",
-    "@platforma-sdk/workflow-tengo": "^2.6.0",
+    "@platforma-sdk/tengo-builder": "^1.17.4",
+    "@platforma-sdk/workflow-tengo": "^2.8.2",
     "@milaboratories/software-pframes-conv": "^2.1.2",
-    "@platforma-open/milaboratories.software-small-binaries": "^1.14.6",
+    "@platforma-open/milaboratories.software-small-binaries": "^1.15.0",
     "@platforma-open/milaboratories.software-mixcr": "4.7.0-133-develop",
-    "@platforma-sdk/test": "^1.17.0",
+    "@platforma-open/milaboratories.software-mitool": "2.3.1-2-main",
+    "@platforma-open/milaboratories.software-paggregate": "^1.0.1",
+    "@platforma-sdk/test": "^1.20.9",
     "vitest": "^2.1.8",
     "typescript": "~5.6.3"
   },

package/src/export-settings.lib.tengo

@@ -444,7 +444,7 @@ shmTreeNodesTableOptions := func(dataDescription) {
 // export data that is unique for clones, but not unique for a node
 // (different clones could be in the same topology node, for example, different time points)
 shmTreeNodesWithClonesTableOptions := func(dataDescription, donorColumn) {
-    donorColumnSpec := donorColumn.get("spec").getDataAsJson()
+    donorColumnSpec := donorColumn.spec
 
     axes := []
     columns := []

package/src/main.tpl.tengo

@@ -35,7 +35,8 @@ wf.body(func(args) {
 		datasets: datasets,
 		donorColumn: donorColumn,
 		params: {
-			downsampling: args.downsampling
+			downsampling: args.downsampling,
+			sequencesOfInterest: args.sequencesOfInterest
 		}
 	})
 
@@ -45,6 +46,9 @@ wf.body(func(args) {
 			"treeNodes": results.output("treeNodes"),
 			"treeNodesWithClones": results.output("treeNodesWithClones"),
 
+			"soiNodesResults": results.output("soiNodesResults"),
+			"soiTreesResults": results.output("soiTreesResults"),
+
 			"tsvs": results.output("tsvs"),
 			"allelesLogs": results.output("allelesLogs"),
 			"treesLogs": results.output("treesLogs"),

package/src/prepare-donor-column.lib.tengo

@@ -20,7 +20,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
 	//        axes[0]: sampleId
 	//          value: fileRef resource
 
-    donorColumnSpec := donorColumn.get("spec").getDataAsJson()
+    donorColumnSpec := donorColumn.spec
 
     sampleIdAxis := donorColumnSpec.axesSpec[0]
 
@@ -53,7 +53,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
     sampleToDonor := {}
 
     // columns with meta could be fetched as data direcctly
-	for k, v in donorColumn.get("data").getDataAsJson()["data"] {
+	for k, v in donorColumn.data.getDataAsJson()["data"] {
         sampleId := json.decode(k)[0]
         sampleToDonor[sampleId] = v
 	}
@@ -63,7 +63,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
 
     // collect all the clns files that we have into pColumn
 	for clonotypingBlockId, dataset in datasets {
-		for sKey, fileRef in dataset.get("data").inputs() {
+		for sKey, fileRef in dataset.data.inputs() {
             sampleId := json.decode(sKey)[0]
             donor := sampleToDonor[sampleId]
             dataBuilder.createInputField(json.encode([donor, sampleId, clonotypingBlockId])).set(fileRef)

package/src/process.tpl.tengo

@@ -4,6 +4,7 @@ llPFrames := import("@platforma-sdk/workflow-tengo:pframes.ll")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 maps := import("@platforma-sdk/workflow-tengo:maps")
 assets := import("@platforma-sdk/workflow-tengo:assets")
+render := import("@platforma-sdk/workflow-tengo:render")
 xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
 text := import("text")
 exportSettings := import(":export-settings")
@@ -20,6 +21,8 @@ self.awaitState("datasets", { wildcard: "*" }, "spec", "ResourceReady")
 self.awaitState("donorColumn", "ResourceReady")
 self.awaitState("params", "ResourceReady")
 
+soiTpl := assets.importTemplate(":soi")
+
 self.body(func(inputs) {
 	// overall description of data that we have.
     dataDescription := {
@@ -35,7 +38,9 @@ self.body(func(inputs) {
 
 	assemblingFeature := ""
 	for clonotypingBlockId, dataset in inputs.datasets {
-		presetAnnotations := dataset.get("spec").getDataAsJson()["annotations"]
+		presetAnnotations := dataset.spec.annotations
+
+		ll.assert(!is_undefined(presetAnnotations), "No annotations in dataset specs")
 
 		datasetTypes[clonotypingBlockId] = "bulk"
 
@@ -149,6 +154,41 @@ self.body(func(inputs) {
         additionalArgsForImportTsv
     )
 
+	// Running SOI search for the data
+	soiNodesResults := {}
+	soiTreesResults := {}
+	for soiDb in inputs.params.sequencesOfInterest {
+
+		columnId := ""
+		if soiDb.parameters.type == "nucleotide" {
+			columnId = "n-seq-"
+		} else if soiDb.parameters.type == "amino-acid" {
+			columnId = "aa-seq-"
+		} else {
+			ll.panic("unknown alphabet: " + soiDb.parameters.type)
+		}
+
+		if soiDb.parameters.targetFeature == "CDR3" {
+			columnId = columnId + "CDR3"
+		} else if soiDb.parameters.targetFeature == "VDJRegion" {
+			columnId = columnId + "VDJRegion"
+		} else {
+			ll.panic("unknown target feature: " + soiDb.parameters.targetFeature)
+		}
+
+		querySpec := treeNodes[columnId + ".spec"]
+		queryData := treeNodes[columnId + ".data"]
+
+		soiResult := render.create(soiTpl, {
+			querySpec: querySpec,
+			queryData: queryData,
+			db: soiDb
+		})
+
+		soiNodesResults[soiDb.parameters.id] = soiResult.output("nodesResult")
+		soiTreesResults[soiDb.parameters.id] = soiResult.output("treesResult")
+	}
+
 	tsvs := mixcrResults.output("tsvs")
 
     return {
@@ -159,6 +199,9 @@ self.body(func(inputs) {
         // combine columns into pFrame
         "treeNodesWithClones": pframes.exportFrame(treeNodesWithClones),
 
+		"soiNodesResults": maps.mapValues(soiNodesResults, pframes.exportFrame),
+		"soiTreesResults": maps.mapValues(soiTreesResults, pframes.exportFrame),
+
         "tsvs": tsvs,
 
 		"allelesLogs": mixcrResults.output("allelesLog"),

package/src/soi-export.lib.tengo

@@ -0,0 +1,39 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+soiResultImportColumns := func(dbParameters) {
+	if dbParameters.searchParameters.type == "tree_search_top" {
+		return [ {
+			"column": "topHit",
+			"id": "topHit",
+			"spec": {
+				"name": "pl7.app/search/topHit",
+				"valueType": "String",
+				"annotations": {
+					"pl7.app/label": "Top Hit " + dbParameters.name
+				}
+			},
+			"domain": {
+				"pl7.app/list": dbParameters.id
+			}
+		}, {
+			"column": "mutations",
+			"id": "mutations",
+			"spec": {
+				"name": "pl7.app/search/numberOfMutations",
+				"valueType": "Int",
+				"annotations": {
+					"pl7.app/label": "Number of mutations " + dbParameters.name
+				}
+			},
+			"domain": {
+				"pl7.app/list": dbParameters.id
+			}
+		} ]
+	} else {
+		ll.panic("Unknown search mode: " + dbParameters.searchParameters.type)
+	}
+}
+
+export ll.toStrict({
+	soiResultImportColumns: soiResultImportColumns
+})

package/src/soi.tpl.tengo

@@ -0,0 +1,152 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
+json := import("json")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+soiExport := import(":soi-export")
+
+self.defineOutputs("nodesResult", "treesResult")
+
+// import MiXCR as a software to use
+mitoolSw := assets.importSoftware("@platforma-open/milaboratories.software-mitool:main")
+paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-paggregate:main")
+
+inferPartitionKeyLength := func(data) {
+	rType := data.info().Type.Name
+	if (rType == pConstants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED.Name) {
+		return data.getDataAsJson().partitionKeyLength
+    } else if (rType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED.Name) {
+		return data.getDataAsJson().partitionKeyLength
+	} else if (rType == pConstants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED.Name) {
+		superPartitionKeyLength := data.getDataAsJson().superPartitionKeyLength
+		partitionKeyLength := data.getDataAsJson().partitionKeyLength
+		return superPartitionKeyLength + partitionKeyLength
+	} else if (rType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED.Name) {
+		superPartitionKeyLength := data.getDataAsJson().superPartitionKeyLength
+		partitionKeyLength := data.getDataAsJson().partitionKeyLength
+		return superPartitionKeyLength + partitionKeyLength
+	}
+	return 0
+}
+
+self.body(func(inputs) {
+	querySpec := inputs.querySpec
+	queryData := inputs.queryData
+	db := inputs.db
+
+	dbData := "sequence\tname\n"
+	for entry in db.sequences {
+		dbData = dbData + entry.sequence + "\t" + entry.name + "\n"
+	}
+
+	originalAxesSpecs := querySpec.axesSpec
+	modifiedAxesSpecs := []
+	importAxesSpec := []
+
+	aggregationGroupByTargets := []
+	aggregationImportAxesSpec := []
+
+	for idx, spec in originalAxesSpecs {
+		colName := "key" + idx
+		modifiedAxesSpecs = append(modifiedAxesSpecs, maps.deepMerge(
+			spec,
+			{ annotations: { "pl7.app/label": colName } }
+		))
+		importAxesSpec = append(importAxesSpec, {
+			"column": colName,
+			"spec": spec
+		})
+
+		// aggregating clonal and subtree axes away
+		if spec.name != "pl7.app/dendrogram/subtreeId" && spec.name != "pl7.app/dendrogram/nodeId" {
+			aggregationGroupByTargets = append(aggregationGroupByTargets, colName)
+			aggregationImportAxesSpec = append(aggregationImportAxesSpec, {
+				"column": colName,
+				"spec": spec
+			})
+		}
+	}
+	modifiedQuerySpec := maps.deepMerge(querySpec, {
+		axesSpec: modifiedAxesSpecs,
+		annotations: {"pl7.app/label": "query"} })
+
+	inputTsv := xsv.exportFrame([{spec: modifiedQuerySpec, data: queryData}], "tsv", {})
+
+	searchCmd := exec.builder().
+        printErrStreamToStdout().
+        secret("MI_LICENSE", "MI_LICENSE").
+        software(mitoolSw).
+        arg("search").
+        arg("--alphabet").arg(db.parameters.type).
+		arg("--database").arg("database.tsv").
+		writeFile("database.tsv", dbData).
+		arg("--parameters").arg("params.json").
+		writeFile("params.json", json.encode(db.parameters.searchParameters)).
+		arg("--hits-only").
+		arg("--target-column").arg("query").
+        arg("input.tsv").addFile("input.tsv", inputTsv).
+		arg("output.tsv").saveFile("output.tsv").
+		run()
+
+	resultCsv := searchCmd.getFile("output.tsv")
+
+	resultColumns := soiExport.soiResultImportColumns(db.parameters)
+	resultConvParams := {
+		"axes": importAxesSpec,
+		"columns": resultColumns,
+		"storageFormat": "Binary",
+		"partitionKeyLength": 0 // inferPartitionKeyLength(queryData)
+	}
+
+	aggregatedConvParams := {
+		"axes": aggregationImportAxesSpec,
+		"columns": resultColumns,
+		"storageFormat": "Binary",
+		"partitionKeyLength": 0 // inferPartitionKeyLength(queryData)
+	}
+
+	aggregations := []
+	for col in resultColumns {
+		aggregations = append(aggregations, {
+			type: "first",
+			src: col.column,
+			dst: col.column
+		})
+	}
+
+	aggregationWorkflow := { steps: [ {
+			type: "aggregate",
+			groupBy: aggregationGroupByTargets,
+			aggregations: aggregations
+		} ] }
+
+	aggregateCmd := exec.builder().
+        printErrStreamToStdout().
+        software(paggregateSw).
+        arg("--workflow").arg("wf.json").
+		writeFile("wf.json", json.encode(aggregationWorkflow)).
+		arg("input.tsv").addFile("input.tsv", resultCsv).
+		arg("output.tsv").saveFile("output.tsv").
+		run()
+
+	aggregatedCsv := aggregateCmd.getFile("output.tsv")
+
+	nodesResult := xsv.importFile(
+        resultCsv,
+        "tsv",
+        resultConvParams
+    )
+
+	treesResult := xsv.importFile(
+        aggregatedCsv,
+        "tsv",
+        aggregatedConvParams
+    )
+
+	return {
+		nodesResult: nodesResult,
+		treesResult: treesResult
+	}
+})