@platforma-open/milaboratories.mixcr-shm-trees.workflow 3.5.0 → 3.6.0

package/.turbo/turbo-build.log

@@ -1,5 +1,6 @@
+ WARN  Issue while reading "/home/runner/work/mixcr-shm-trees/mixcr-shm-trees/.npmrc". Failed to replace env in config: ${NPMJS_TOKEN}
 
-> @platforma-open/milaboratories.mixcr-shm-trees.workflow@3.5.0 build /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow
+> @platforma-open/milaboratories.mixcr-shm-trees.workflow@3.6.0 build /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow
 > rm -rf dist && pl-tengo check && pl-tengo build
 
 Processing "src/export-settings.lib.tengo"...
@@ -7,6 +8,7 @@ Processing "src/main.tpl.tengo"...
 Processing "src/prepare-donor-column.lib.tengo"...
 Processing "src/process.tpl.tengo"...
 Processing "src/reconstruct-shm-trees.tpl.tengo"...
+Processing "src/request-library.tpl.tengo"...
 Processing "src/soi-export.lib.tengo"...
 Processing "src/soi.tpl.tengo"...
 Processing "src/tablesAggregation.lib.tengo"...
@@ -17,8 +19,9 @@ No syntax errors found.
   info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/lib/soi-export.lib.tengo
   info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/lib/tablesAggregation.lib.tengo
   info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/reconstruct-shm-trees.plj.gz
+  info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/request-library.plj.gz
   info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/soi.plj.gz
   info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/process.plj.gz
   info:   - writing /home/runner/work/mixcr-shm-trees/mixcr-shm-trees/workflow/dist/tengo/tpl/main.plj.gz
-  info: 
+  info: Template Pack build done.
   info: Template Pack build done.

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @platforma-open/milaboratories.mixcr-shm-trees.workflow
 
+## 3.6.0
+
+### Minor Changes
+
+- b8af059: SDK Upgrade
+- b8af059: Custom library support
+
+## 3.5.1
+
+### Patch Changes
+
+- 089219e: dependency upgrade
+
 ## 3.5.0
 
 ### Minor Changes

package/dist/index.cjs

@@ -1,5 +1,6 @@
 module.exports = { Templates: {
   'reconstruct-shm-trees': { type: 'from-file', path: require.resolve('./tengo/tpl/reconstruct-shm-trees.plj.gz') },
+  'request-library': { type: 'from-file', path: require.resolve('./tengo/tpl/request-library.plj.gz') },
   'soi': { type: 'from-file', path: require.resolve('./tengo/tpl/soi.plj.gz') },
   'process': { type: 'from-file', path: require.resolve('./tengo/tpl/process.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "reconstruct-shm-trees" | "soi" | "process" | "main";
+declare type TplName = "reconstruct-shm-trees" | "request-library" | "soi" | "process" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,6 +1,7 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'reconstruct-shm-trees': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/reconstruct-shm-trees.plj.gz') },
+  'request-library': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/request-library.plj.gz') },
   'soi': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/soi.plj.gz') },
   'process': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/process.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }

package/dist/tengo/lib/export-settings.lib.tengo

@@ -1,18 +1,18 @@
 ll := import("@platforma-sdk/workflow-tengo:ll")
 text := import("text")
 
-// ==============================================
-//
-// Every function in this file will return:
-//   pfconvParams - params to run xsv.importFileMap on exported files
-//   cmdArgs - additional args for MiXCR to specify what fields to extract
-//
-// ==============================================
-
-// TODO: TMP
+
+
+
+
+
+
+
+
+
 allowNA := true
 
-// export of threes whithout nodes
+
 shmTreeTableOptions := func(dataDescription) {
     axes := []
     columns := []
@@ -300,7 +300,7 @@ shmTreeTableOptions := func(dataDescription) {
     }
 }
 
-// export data that is uniq for node
+
 shmTreeNodesTableOptions := func(dataDescription) {
     axes := []
     columns := []
@@ -361,7 +361,7 @@ shmTreeNodesTableOptions := func(dataDescription) {
         allowNA: false,
         spec: {
             name: "pl7.app/dendrogram/isObserved",
-            // TODO change to Boolean when it will be supported
+
             valueType: "String",
             annotations: {
                 "pl7.app/label": "Is observed in data",
@@ -811,8 +811,8 @@ shmTreeNodesTableOptions := func(dataDescription) {
     }
 }
 
-// export data that is unique for clones, but not unique for a node
-// (different clones could be in the same topology node, for example, different time points)
+
+
 shmTreeNodesWithClonesTableOptions := func(dataDescription, donorColumn) {
     donorColumnSpec := donorColumn.spec
 
@@ -870,8 +870,8 @@ shmTreeNodesWithClonesTableOptions := func(dataDescription, donorColumn) {
     cmdArgs = append(cmdArgs, "-fileName")
     axes = append(axes, {
         column: "fileName",
-        // in case of single cell trees, not all subtrees will be filled for nodes with observed
-        // allowNA: true,
+
+
         preProcess: [
             {
                 type: "regexpReplace",
@@ -892,13 +892,13 @@ shmTreeNodesWithClonesTableOptions := func(dataDescription, donorColumn) {
     axes = append(axes, {
         column: "cloneId",
         id: "clone-id",
-        // allowNA: true,
-        // filtering off records without clones because we are not interested in them here
+
+
         filterOutRegex: "^$",
         spec: {
             name: "pl7.app/vdj/cloneId",
             type: "Long",
-            // TODO domain with blockId
+
             annotations: {
                 "pl7.app/label": "Clone id",
                 "pl7.app/table/visibility": "default",
@@ -1135,7 +1135,7 @@ shmTreeNodesUniqueIsotypeTableOptions := func(dataDescription) {
     }
 }
 
-// to use the file as a library, we should explicitly export functions
+
 export ll.toStrict({
 	shmTreeTableOptions: shmTreeTableOptions,
     shmTreeNodesTableOptions: shmTreeNodesTableOptions,

package/dist/tengo/lib/prepare-donor-column.lib.tengo

@@ -7,19 +7,19 @@ json := import("json")
 _P_COLUMN_DATA_RESOURCE_MAP := { Name: "PColumnData/ResourceMap", Version: "1" }
 
 groupDataByDonorId := func(donorColumn, datasets) {
-    // we need to form a pColumn with two axes:
-	//    axes[0]: donorId
-	//    axes[1]: sampleId
-	//    axes[2]: mixcrclonotypingBlockId
-	//      value: fileRef resource
-
-	// we have:
-	//    column of donorIds:
-	//        axes[0]: sampleId
-	//          value: donorId
-	//    several columns of clns:
-	//        axes[0]: sampleId
-	//          value: fileRef resource
+
+
+
+
+
+
+
+
+
+
+
+
+
 
     donorColumnSpec := donorColumn.spec
 
@@ -30,7 +30,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
         name: "mixcr.com/clns",
         valueType: "File",
 
-        // annotations and domain could differ between datasets
+
         axesSpec: [
             {
                 name: donorColumnSpec.name,
@@ -49,20 +49,20 @@ groupDataByDonorId := func(donorColumn, datasets) {
         ]
     }, { removeUndefs: true })
 
-    // creating sample to donor map
+
 
     sampleToDonor := {}
 
-    // columns with meta could be fetched as data directly
+
 	for k, v in donorColumn.data.getDataAsJson()["data"] {
         sampleId := json.decode(k)[0]
         sampleToDonor[sampleId] = v
 	}
 
-    // build pColumn by hand
+
     dataBuilder := smart.structBuilder(_P_COLUMN_DATA_RESOURCE_MAP, json.encode({ keyLength: 3 }))
 
-    // collect all the clns files that we have into pColumn
+
 	for clonotypingBlockId, dataset in datasets {
 		for sKey, fileRef in dataset.data.inputs() {
             sampleId := json.decode(sKey)[0]
@@ -77,7 +77,7 @@ groupDataByDonorId := func(donorColumn, datasets) {
     }
 }
 
-// to use the file as a library, we should explicitly export functions
+
 export ll.toStrict({
 	groupDataByDonorId: groupDataByDonorId
 })

package/dist/tengo/lib/tablesAggregation.lib.tengo

@@ -4,7 +4,7 @@ assets := import("@platforma-sdk/workflow-tengo:assets")
 json := import("json")
 paggregateSw := assets.importSoftware("@platforma-open/milaboratories.software-ptransform:main")
 
-//// aggregating by-nodes output to make it uniquely addressable by it's native key
+
 ensureUniqueness := func(inputTsv, pfConvParams, ...aggParams) {
     keyColumns := []
     for axis in pfConvParams.axes {

package/package.json

@@ -1,21 +1,21 @@
 {
   "name": "@platforma-open/milaboratories.mixcr-shm-trees.workflow",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "type": "module",
   "description": "Tengo-based template",
   "//": {
     "build": "node ./scripts/build-static.mjs src/pfconv_params.json src/pfconv_params.lib.tengo && rm -rf dist && pl-tengo check && pl-tengo build && ./create_tags.sh"
   },
+  "dependencies": {
+    "@platforma-sdk/workflow-tengo": "^4.2.0"
+  },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^1.17.5",
-    "@platforma-sdk/workflow-tengo": "^2.11.0",
-    "@milaboratories/software-pframes-conv": "^2.1.9",
-    "@platforma-open/milaboratories.software-small-binaries": "^1.15.7",
+    "@platforma-sdk/tengo-builder": "^2.1.3",
     "@platforma-open/milaboratories.software-mixcr": "4.7.0-139-develop",
     "@platforma-open/milaboratories.software-mitool": "2.3.1-5-main",
-    "@platforma-open/milaboratories.software-ptransform": "^1.1.1",
-    "@platforma-sdk/test": "^1.22.40",
-    "vitest": "^2.1.8",
+    "@platforma-open/milaboratories.software-ptransform": "^1.3.1",
+    "@platforma-sdk/test": "^1.30.4",
+    "vitest": "~2.1.8",
     "typescript": "~5.6.3"
   },
   "scripts": {

package/src/main.tpl.tengo

@@ -6,6 +6,7 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 pframes := import("@platforma-sdk/workflow-tengo:pframes")
 
 processTpl := assets.importTemplate(":process")
+requestLibraryTpl := assets.importTemplate(":request-library")
 
 wf.body(func(args) {
 	if is_undefined(args.donorColumn) {
@@ -19,13 +20,15 @@ wf.body(func(args) {
 	// we could not use array as request for waiting (see below), so we store datasets in a dictionary
 	datasets := {}
 	for datasetRef in args.datasetColumns {
-		if is_undefined(datasetRef) {
-			ll.panic("Dataset is undefined")
-		}
-		// it's blockId of MiXCR, we suppose that it procuce only one clns column
+		// we assume that mixcr block produces exactly one clns column
 		datasets[datasetRef.blockId] = wf.resolve(datasetRef)
 	}
 
+	library := render.createEphemeral(requestLibraryTpl, {
+		datasets: datasets,
+		ctx: wf.getParentBCtx()
+	}).output("library")
+
 	donorColumn := wf.resolve(args.donorColumn)
 
 	// The problem is that refs for data is not resolved.
@@ -34,6 +37,7 @@ wf.body(func(args) {
 	results := render.createEphemeral(processTpl, {
 		datasets: datasets,
 		donorColumn: donorColumn,
+		library: library,
 		params: {
 			downsampling: args.downsampling,
 			sequencesOfInterest: args.sequencesOfInterest

package/src/process.tpl.tengo

@@ -6,6 +6,7 @@ maps := import("@platforma-sdk/workflow-tengo:maps")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 render := import("@platforma-sdk/workflow-tengo:render")
 xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
+smart := import("@platforma-sdk/workflow-tengo:smart")
 text := import("text")
 json := import("json")
 exportSettings := import(":export-settings")
@@ -20,6 +21,7 @@ self.awaitState("datasets", { wildcard: "*" }, "AllInputsSet")
 self.awaitState("datasets", { wildcard: "*" }, "data", "InputsLocked")
 // but we need spec already
 self.awaitState("datasets", { wildcard: "*" }, "spec", "ResourceReady")
+self.awaitState("library", "spec", "ResourceReady")
 self.awaitState("donorColumn", "ResourceReady")
 self.awaitState("params", "ResourceReady")
 
@@ -36,6 +38,8 @@ self.body(func(inputs) {
 		cellsAssembled: false
 	}
 
+	library := inputs.library
+
 	// clonotypingBlockId -> "bulk" | "sc"
 	datasetTypes := {}
 
@@ -78,6 +82,8 @@ self.body(func(inputs) {
 	shmTreeNodesWithClonesTableOptions := exportSettings.shmTreeNodesWithClonesTableOptions(dataDescription, inputs.donorColumn)
 	shmTreeNodesUniqueIsotypeTableOptions := exportSettings.shmTreeNodesUniqueIsotypeTableOptions(dataDescription)
 
+	libraryFormat := library.spec.annotations["pl7.app/vdj/libraryFormat"]
+
 	// TODO that call is too low level. Should be replaced with something that works with pColumns, not data only
 	mixcrResults := llPFrames.aggregate(
 		// files to iterate through
@@ -131,9 +137,13 @@ self.body(func(inputs) {
 			shmTreeNodesWithClonesTableOptions: shmTreeNodesWithClonesTableOptions,
 			shmTreeNodesWithClonesTableArgs: shmTreeNodesWithClonesTableOptions.cmdArgs,
 			shmTreeNodesUniqueIsotypeTableOptions: shmTreeNodesUniqueIsotypeTableOptions,
+			library: is_undefined(library) ? smart.createNullResource() : library.data,
 			globalParams: maps.merge(
 				inputs.params,
-				{ datasetTypes: datasetTypes }
+				{
+					datasetTypes: datasetTypes,
+					libraryFormat: libraryFormat
+				}
 			)
 		}
 	)
@@ -143,7 +153,7 @@ self.body(func(inputs) {
 		additionalAxesSpec: dataGroupedByDonorId.spec["axesSpec"][:1]
 	}
 
-	ll.print("__THE_LOG__ " + string(json.encode(additionalArgsForImportTsv)))
+	// ll.print("__THE_LOG__ " + string(json.encode(additionalArgsForImportTsv)))
 
 	trees := xsv.importFileMap(
         mixcrResults.output("trees"),

package/src/reconstruct-shm-trees.tpl.tengo

@@ -33,19 +33,32 @@ self.body(func(inputs) {
     globalParams := inputs.globalParams
     datasetTypes := globalParams.datasetTypes
     downsampling := globalParams.downsampling
+    library := inputs.library
+    libraryFormat := globalParams.libraryFormat
 
     // ll.print("__THE_LOG__ " + json.encode(datasetTypes))
     // ll.print("__THE_LOG__ " + json.encode(downsampling))
+    // ll.print("__THE_LOG__ " + json.encode(libraryFormat))
 
     ll.assert(!is_undefined(datasetTypes), "datasetTypes undefined")
 
-    seed := times.time_string(times.now())
+    addLibraryFile := func(cmdBuilder) {
+        if !is_undefined(library) {
+            if libraryFormat == "repseqio.json.gz" {
+                cmdBuilder.addFile("library.json.gz", library)
+            } else {
+                cmdBuilder.addFile("library.json", library)
+            }
+        }
+    }
+
+    // seed := times.time_string(times.now())
 
     allelesCmdBuilder := exec.builder().
         printErrStreamToStdout().
         secret("MI_LICENSE", "MI_LICENSE").
         env("MI_PROGRESS_PREFIX", progressPrefix).
-        env("SEED", seed).
+        // env("SEED", seed).
         software(mixcrSw).
         arg("findAlleles").
         arg("--report").arg("report.txt").
@@ -55,6 +68,8 @@ self.body(func(inputs) {
         // template specifies where result files will be written
         arg("--output-template").arg("alleles/{file_name}.clns")
 
+    addLibraryFile(allelesCmdBuilder)
+
     if !is_undefined(globalParams.seed) {
         allelesCmdBuilder.env("SEED", globalParams.seed)
     }
@@ -108,7 +123,7 @@ self.body(func(inputs) {
 
         for input in toProcess {
             if datasetTypes[input.clonotypingBlockId] == "bulk" {
-                downsamplingCmd := exec.builder().
+                downsamplingCmdBuilder := exec.builder().
                     printErrStreamToStdout().
                     secret("MI_LICENSE", "MI_LICENSE").
                     env("MI_PROGRESS_PREFIX", progressPrefix).
@@ -118,8 +133,9 @@ self.body(func(inputs) {
                     arg(downsamplingParam).
                     arg("clones.clns").
                     addFile("clones.clns", input.alleles).
-                    saveFile("clones.downsampled.clns").
-                    run()
+                    saveFile("clones.downsampled.clns")
+                addLibraryFile(downsamplingCmdBuilder)
+                downsamplingCmd := downsamplingCmdBuilder.run()
                 input.alleles = downsamplingCmd.getFile("clones.downsampled.clns")
             }
         }
@@ -129,7 +145,6 @@ self.body(func(inputs) {
         printErrStreamToStdout().
         secret("MI_LICENSE", "MI_LICENSE").
         env("MI_PROGRESS_PREFIX", progressPrefix).
-        env("SEED", seed).
         software(mixcrSw).
         arg("findShmTrees").
         arg("--report").arg("report.txt").
@@ -141,6 +156,8 @@ self.body(func(inputs) {
         shmTreesCmdBuilder.env("SEED", globalParams.seed)
     }
 
+    addLibraryFile(shmTreesCmdBuilder)
+
     for input in toProcess {
         shmTreesCmdBuilder.
             addFile(input.fileName, input.alleles).
@@ -151,15 +168,18 @@ self.body(func(inputs) {
 
     shmTrees := shmTreesCmdBuilder.run()
     outputShmt := shmTrees.getFile("output.shmt")
-    
+
     // export trees without nodes
     shmTreeExportsCmdBuilder := exec.builder().
         printErrStreamToStdout().
+        dontSaveStdoutOrStderr().
         inLightQueue().
         secret("MI_LICENSE", "MI_LICENSE").
         software(mixcrSw).
         arg("exportShmTrees")
 
+    addLibraryFile(shmTreeExportsCmdBuilder)
+
     for arg in inputs.shmTreeTableArgs {
         shmTreeExportsCmdBuilder = shmTreeExportsCmdBuilder.arg(arg)
     }
@@ -178,11 +198,14 @@ self.body(func(inputs) {
     // export tree nodes with data uniq for the node
     shmTreeNodesExportsCmdBuilder := exec.builder().
         printErrStreamToStdout().
+        dontSaveStdoutOrStderr().
         inLightQueue().
         secret("MI_LICENSE", "MI_LICENSE").
         software(mixcrSw).
         arg("exportShmTreesWithNodes")
 
+    addLibraryFile(shmTreeNodesExportsCmdBuilder)
+
     for arg in inputs.shmTreeNodesTableOptions.cmdArgs {
         shmTreeNodesExportsCmdBuilder = shmTreeNodesExportsCmdBuilder.arg(arg)
     }
@@ -201,6 +224,7 @@ self.body(func(inputs) {
     // export nodes with clones. For each node could be several clones
     shmTreeNodesWithClonesExportsCmdBuilder := exec.builder().
         printErrStreamToStdout().
+        dontSaveStdoutOrStderr().
         inLightQueue().
         secret("MI_LICENSE", "MI_LICENSE").
         env("MI_PROGRESS_PREFIX", progressPrefix).
@@ -209,6 +233,8 @@ self.body(func(inputs) {
         // don't export nodes that don't have clones
         arg("--only-observed")
 
+    addLibraryFile(shmTreeNodesWithClonesExportsCmdBuilder)
+
     for arg in inputs.shmTreeNodesWithClonesTableArgs {
         shmTreeNodesWithClonesExportsCmdBuilder = shmTreeNodesWithClonesExportsCmdBuilder.arg(arg)
     }

package/src/request-library.tpl.tengo

@@ -0,0 +1,42 @@
+smart := import("@platforma-sdk/workflow-tengo:smart")
+bquery := import("@platforma-sdk/workflow-tengo:workflow.bquery")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+self := import("@platforma-sdk/workflow-tengo:tpl")
+
+self.awaitState("datasets", { wildcard: "*" }, "spec", "ResourceReady")
+self.awaitState("datasets", { wildcard: "*" }, "ref", "ResourceReady")
+
+self.body(func(args) {
+	datasets := args.datasets
+	ctx := args.ctx
+
+	// undefined - not initialized
+	// "" - no custom library, built-in library will be used
+	// "...." - custom library id, to be requested from the context
+	libraryId := undefined
+
+	for _, dataset in datasets {
+		dsLibraryId := dataset.spec.annotations["pl7.app/vdj/libraryId"]
+		if is_undefined(libraryId) {
+			libraryId = !is_undefined(dsLibraryId) ? dsLibraryId : ""
+		} else {
+			if libraryId != dsLibraryId {
+				ll.panic("All datasets should have the same libraryId. Got " + libraryId + " and " + dsLibraryId)
+			}
+		}
+	}
+
+	if libraryId == "" || is_undefined(libraryId) {
+		return { library: smart.createNullResource() }
+	} else {
+		querySpec := {
+			type: "And",
+			operands: [
+				{ type: "Name", name: "pl7.app/vdj/library" },
+				{ type: "Domain", domain: { "pl7.app/vdj/libraryId": libraryId } }
+			]
+		}
+		return { library: bquery.create(querySpec, ctx, { single: true, errIfMissing: true }) }
+	}
+})
+