@platforma-open/milaboratories.mixcr-clonotyping-2.workflow 2.0.0

package/src/process.tpl.tengo

@@ -0,0 +1,337 @@
+// process
+
+self := import("@platforma-sdk/workflow-tengo:tpl")
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+pframes := import("@platforma-sdk/workflow-tengo:pframes")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+file := import("@platforma-sdk/workflow-tengo:file")
+llPFrames := import("@platforma-sdk/workflow-tengo:pframes.ll")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+pUtil := import("@platforma-sdk/workflow-tengo:pframes.util")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+
+calculateExportSpecs := import(":calculate-export-specs")
+
+json := import("json")
+text := import("text")
+
+mixcrAnalyzeTpl := assets.importTemplate(":mixcr-analyze")
+mixcrExportTpl := assets.importTemplate(":mixcr-export")
+aggregateByClonotypeKeyTpl := assets.importTemplate(":aggregate-by-clonotype-key")
+
+self.awaitState("InputsLocked")
+self.awaitState("params", "ResourceReady")
+self.awaitState("inputSpec", "ResourceReady")
+self.awaitState("presetSpecForBack", "ResourceReady")
+self.awaitState("presetContent", "ResourceReady")
+
+self.body(func(inputs) {
+
+	inputSpec := inputs.inputSpec
+	library := inputs.library
+
+	preset := inputs.preset
+	presetSpecForBack := inputs.presetSpecForBack.getDataAsJson()
+	presetContent := inputs.presetContent
+
+	params := inputs.params
+	species := params.species
+	chains := params.chains
+	limitInput := params.limitInput
+	blockId := params.blockId
+	presetCommonName := params.presetCommonName
+	isLibraryFileGzipped := params.isLibraryFileGzipped
+
+
+	if is_undefined(presetSpecForBack) {
+		ll.panic("no presetSpecForBack")
+	}
+
+	reports := []
+
+	for step in presetSpecForBack.reportTypes {
+		if step == "align" || step == "assemble" {
+			reports = append(reports, {
+				id: step,
+				fileJson: "result." + step + ".report.json",
+				fileTxt: "result." + step + ".report.txt"
+			})
+		}
+	}
+
+	hasAssembleContigs := false
+	hasAssembleCells := false
+	for stage in presetSpecForBack.analysisStages {
+		if stage == "assembleContigs" {
+			hasAssembleContigs = true
+		} else if stage == "assembleCells" {
+			hasAssembleCells = true
+		}
+	}
+
+	// calculating clns annotations
+
+	joinOrUndefined := func(arr) {
+		if is_undefined(arr) {
+			return undefined
+		} else {
+			return text.join(arr, ",")
+		}
+	}
+
+	removeUndefined := func(m) {
+		r := {}
+		for k, v in m {
+			if !is_undefined(v) {
+				r[k] = v
+			}
+		}
+		return r;
+	}
+
+	clnsAnnotations := removeUndefined({
+		"mixcr.com/assemblingFeature": joinOrUndefined(presetSpecForBack.assemblingFeature),
+		"mixcr.com/cellTags": joinOrUndefined(presetSpecForBack.cellTags),
+		"mixcr.com/coveredFeaturesOnExport": joinOrUndefined(presetSpecForBack.coveredFeaturesOnExport),
+		"mixcr.com/umiTags": joinOrUndefined(presetSpecForBack.umiTags),
+		"pl7.app/label": "MiXCR Clonesets"
+	})
+	if hasAssembleContigs {
+		clnsAnnotations["mixcr.com/contigsAssembled"] = "true"
+	}
+	if hasAssembleCells {
+		clnsAnnotations["mixcr.com/cellsAssembled"] = "true"
+	}
+
+	fileExtension := inputSpec.domain["pl7.app/fileExtension"]
+
+	targetOutputs := [ {
+		type: "Resource",
+		spec: {
+			kind: "PColumn",
+			valueType: "File",
+			name: "mixcr.com/qc",
+			domain: {
+				"pl7.app/blockId": blockId
+			}
+		},
+		name: "qc"
+	}, {
+		type: "Resource",
+		spec: {
+			kind: "PColumn",
+			name: "pl7.app/log",
+			domain: {
+				"pl7.app/blockId": blockId
+			},
+			valueType: "Log"
+		},
+		name: "log"
+	}, {
+		type: "Resource",
+		spec: {
+			kind: "PColumn",
+			name: "mixcr.com/clns",
+			domain: {
+				"pl7.app/blockId": blockId
+			},
+			annotations: clnsAnnotations,
+			valueType: "File"
+		},
+		name: "clns"
+	}, {
+		type: "ResourceMap",
+		name: "reports",
+		spec: {
+			kind: "PColumn",
+			name: "mixcr.com/report",
+			domain: {
+				"pl7.app/blockId": blockId
+			},
+			valueType: "File",
+			axesSpec: [ {
+				type: "String",
+				name: "mixcr.com/report/source",
+				annotations: {
+					"pl7.app/label": "Source MiXCR stage"
+				}
+			}, {
+				type: "String",
+				name: "mixcr.com/report/format",
+				annotations: {
+					"pl7.app/label": "Report format"
+				},
+				domain: {
+					"pl7.app/dense": string(json.encode(["json", "txt"]))
+				}
+			} ]
+		}
+	} ]
+
+	exportSpecs := calculateExportSpecs(presetSpecForBack, blockId)
+
+	columnsSpecPerSample := exportSpecs.columnsSpecPerSample
+	columnsSpecPerClonotype := exportSpecs.columnsSpecPerClonotype
+	columnsSpec := exportSpecs.columnsSpec
+
+	clonotypeKeyColumns := exportSpecs.clonotypeKeyColumns
+
+	axesByClonotypeId := exportSpecs.axesByClonotypeId
+	axesByClonotypeKey := exportSpecs.axesByClonotypeKey
+
+	exportArgs := exportSpecs.exportArgs
+
+	mainAbundanceColumn := exportSpecs.mainAbundanceColumn
+
+	mixcrResults := pframes.processColumn(
+		{ spec: inputSpec, data: inputs.inputData },
+		mixcrAnalyzeTpl,
+		targetOutputs,
+		{
+			aggregate: [{
+				name: "pl7.app/sequencing/lane",
+				optional: true
+			}, {
+				name: "pl7.app/sequencing/readIndex",
+				optional: true
+			}],
+			// resulting aggregation axes names will be checked against supported combinations
+			// in the body template
+			passAggregationAxesNames: true,
+
+			// will be automatically propagated to all output specs
+			traceSteps: [{type: "milaboratories.mixcr-clonotyping", id: blockId, importance: 20, label: "MiXCR " + presetCommonName}],
+
+			extra: {
+				preset: preset,
+				params: {
+					species: species,
+					limitInput: limitInput,
+					fileExtension: fileExtension,
+					reports: reports,
+					isLibraryFileGzipped: isLibraryFileGzipped
+				},
+				library: library,
+				presetContent: presetContent
+			}
+		}
+	)
+
+	clnsFiles := mixcrResults.output("clns")
+
+	clonotypes := pframes.pFrameBuilder()
+
+	for chain in chains {
+		exportOutputs := [ {
+			type: "Resource",
+			spec: {
+				kind: "PColumn",
+				name: "mixcr.com/clonotypeTable",
+				domain: {
+					"pl7.app/blockId": blockId
+				},
+				valueType: "File"
+			},
+			name: "clonotypeTable",
+			path: ["tsv"]
+		// }, {
+		// 	type: "Xsv",
+		// 	xsvType: "tsv",
+		// 	settings: {
+		// 		axes: axesByClonotypeId,
+		// 		columns: columnsSpec,
+		// 		storageFormat: "Binary",
+		// 		partitionKeyLength: 0
+		// 	},
+		// 	name: "byCloneId",
+		// 	path: ["tsv"]
+		} ]
+
+		if !is_undefined(axesByClonotypeKey) {
+			exportOutputs += [ {
+				type: "Xsv",
+				xsvType: "tsv",
+				settings: {
+					axes: axesByClonotypeKey,
+					columns: columnsSpecPerSample,
+					storageFormat: "Binary",
+					partitionKeyLength: 0
+				},
+				name: "byCloneKeyBySample",
+				path: ["tsv"]
+			} ]
+		}
+
+		exportResults := pframes.processColumn(
+			clnsFiles,
+			mixcrExportTpl,
+			exportOutputs,
+			{
+				// will be automatically propagated to all output specs
+				traceSteps: [{type: "milaboratories.mixcr-clonotyping.export", id: blockId + "." + chain.name, importance: 80, label: chain.name}],
+
+				extra: {
+					params: {
+						chains: chain.mixcrChain,
+						clonotypeKeyColumns: clonotypeKeyColumns,
+						exportArgs: exportArgs
+					}
+				}
+			}
+		)
+
+		// exportResults.addXsvOutputToBuilder(clonotypes, "byCloneId", "byCloneId/" + chain.name + "/")
+		exportResults.addXsvOutputToBuilder(clonotypes, "byCloneKeyBySample", "byCloneKeyBySample/" + chain.name + "/")
+
+		aggregationOutputs := [ {
+			type: "Xsv",
+			xsvType: "tsv",
+			settings: {
+				axes: axesByClonotypeKey,
+				columns: columnsSpecPerClonotype,
+				storageFormat: "Binary",
+				partitionKeyLength: 0
+			},
+			name: "byCloneKey",
+			path: ["tsv"]
+		} ]
+
+		aggregateByCloneKey := pframes.processColumn(
+			exportResults.output("clonotypeTable"),
+			aggregateByClonotypeKeyTpl,
+			aggregationOutputs,
+			{
+				aggregate: ["pl7.app/sampleId"],
+				extra: {
+					params: {
+						mainAbundanceColumn: mainAbundanceColumn,
+						clonotypeColumns: slices.map(columnsSpecPerClonotype, func(col) {
+							return col.column
+						})
+					}
+				}
+			}
+		)
+
+		aggregateByCloneKey.addXsvOutputToBuilder(clonotypes, "byCloneKey", "byCloneKey/" + chain.name + "/")
+	}
+
+	return {
+		"qc.spec": mixcrResults.outputSpec("qc"),
+		"qc.data": mixcrResults.outputData("qc"),
+
+		"logs.spec": mixcrResults.outputSpec("log"),
+		"logs.data": mixcrResults.outputData("log"),
+
+		"reports.spec": mixcrResults.outputSpec("reports"),
+		"reports.data": mixcrResults.outputData("reports"),
+
+		"clns.spec": mixcrResults.outputSpec("clns"),
+		"clns.data": mixcrResults.outputData("clns"),
+
+		clonotypes: clonotypes.build()
+	}
+})

package/src/test/columns-calculate.tpl.tengo

@@ -0,0 +1,16 @@
+self := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+calculateExportSpecs := import(":calculate-export-specs")
+
+self.defineOutputs("exportSpecs")
+
+self.body(func(inputs) {
+	presetSpecForBack := inputs.presetSpecForBack.getDataAsJson()
+
+	exportSpecs := calculateExportSpecs(presetSpecForBack, "test")
+
+	return {
+		exportSpecs: exportSpecs
+	}
+})

package/src/test/columns.test.tpl.tengo

@@ -0,0 +1,28 @@
+// columns-test
+
+self := import("@platforma-sdk/workflow-tengo:tpl")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+render := import("@platforma-sdk/workflow-tengo:render")
+
+calculatePresetInfoTpl := assets.importTemplate(":calculate-preset-info")
+columnsCalculateTpl := assets.importTemplate(":test.columns-calculate")
+
+self.defineOutputs("conf")
+
+self.body(func(inputs) {
+	preset := inputs.preset
+	params := inputs.params
+
+	getPreset := render.create(calculatePresetInfoTpl, {
+		preset: preset,
+		params: params
+	})
+
+	calumnsCalculateResult := render.create(columnsCalculateTpl, {
+		presetSpecForBack: getPreset.output("presetSpecForBack")
+	})
+
+	return {
+		exportSpecs: calumnsCalculateResult.output("exportSpecs")
+	}
+})

package/src/test/columns.test.ts

@@ -0,0 +1,146 @@
+import { ImportFileHandle } from '@platforma-sdk/model';
+import { awaitStableState, tplTest, ML } from '@platforma-sdk/test';
+import { ExpectStatic } from 'vitest';
+
+type Preset =
+  | {
+      type: 'name';
+      name: string;
+    }
+  | {
+      type: 'file';
+      file: ImportFileHandle;
+    };
+
+type Params = {
+  species?: string;
+};
+
+type TestCase = {
+  preset: string;
+  species?: string;
+  check: (expect: ExpectStatic, config: any) => void;
+};
+
+const testCases: TestCase[] = [
+  {
+    preset: 'milab-human-dna-xcr-7genes-multiplex',
+    check: (expect, config) => {
+      // console.dir(config, { depth: 5 });
+      expect(config.axesByClonotypeId).to.have.lengthOf(1);
+      expect(config.axesByClonotypeId.find((c: any) => c.column === 'cloneId')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'readCount')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'readFraction')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR3')).toBeDefined();
+    }
+  },
+  {
+    preset: '10x-sc-xcr-vdj',
+    species: 'human',
+    check: (expect, config) => {
+      // console.dir(config, { depth: 5 });
+      expect(config.axesByClonotypeId).to.have.lengthOf(2);
+      expect(config.axesByClonotypeId.find((c: any) => c.column === 'tagValueCELL')).toBeDefined();
+      expect(config.axesByClonotypeId.find((c: any) => c.column === 'cloneId')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'cellGroup')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'uniqueMoleculeCount')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'uniqueMoleculeFraction')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR4')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR4')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'topChains')).toBeDefined();
+    }
+  },
+  {
+    preset: 'cellecta-human-rna-xcr-umi-drivermap-air',
+    check: (expect, config) => {
+      // console.dir(config, { depth: 5 });
+      expect(config.axesByClonotypeId).to.have.lengthOf(1);
+      expect(config.axesByClonotypeId.find((c: any) => c.column === 'cloneId')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'uniqueMoleculeCount')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'uniqueMoleculeFraction')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR3')).toBeDefined();
+    }
+  },
+  {
+    preset: 'takara-human-rna-bcr-umi-smartseq',
+    check: (expect, config) => {
+      // console.dir(config, { depth: 5 });
+      expect(config.axesByClonotypeId).to.have.lengthOf(1);
+      expect(config.axesByClonotypeId.find((c: any) => c.column === 'cloneId')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'uniqueMoleculeCount')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'uniqueMoleculeFraction')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqFR4')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR1')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR2')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqFR4')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'isotypePrimary')).toBeDefined();
+    }
+  },
+  {
+    preset: 'rna-seq',
+    species: 'human',
+    check: (expect, config) => {
+      // console.dir(config, { depth: 5 });
+      expect(config.axesByClonotypeId).to.have.lengthOf(1);
+      expect(config.axesByClonotypeId.find((c: any) => c.column === 'cloneId')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'readCount')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'readFraction')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'nSeqCDR3')).toBeDefined();
+      expect(config.columnsSpec.find((c: any) => c.column === 'aaSeqCDR3')).toBeDefined();
+    }
+  },
+  {
+    preset: 'generic-single-cell-gex',
+    species: 'human',
+    check: (expect, config) => {
+      // console.dir(config, { depth: 5 });
+      expect(config.axesByClonotypeId).to.have.lengthOf(1);
+      expect(config.columnsSpec.find((c: any) => c.column === 'readCount')).toBeDefined();
+    }
+  }
+];
+
+tplTest.for(testCases)(
+  'checking preset for $preset',
+  { timeout: 30000 },
+  async ({ preset, species, check }, { helper, expect }) => {
+    const resultC = (
+      await helper.renderTemplate(true, 'test.columns.test', ['exportSpecs'], (tx) => {
+        return {
+          preset: tx.createValue(
+            ML.Pl.JsonObject,
+            JSON.stringify({ type: 'name', name: preset } satisfies Preset)
+          ),
+          params: tx.createValue(ML.Pl.JsonObject, JSON.stringify({ species } satisfies Params))
+        };
+      })
+    ).computeOutput('exportSpecs', (c) => c?.getDataAsJson());
+    const result = await awaitStableState(resultC, 20000);
+    // console.dir(result, { depth: 5 });
+    check(expect, result);
+  }
+);

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "strict": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "sourceMap": true,
+    "declaration": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/vitest.config.mts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    watch: false,
+    maxConcurrency: 3,
+    testTimeout: 10000,
+    retry: 2
+  }
+});