@platforma-open/milaboratories.samples-and-data.workflow 1.13.2 → 2.0.0

package/dist/index.cjs

@@ -1,3 +1,5 @@
 module.exports = { Templates: {
+  'parse-bulk-count-matrix': { type: 'from-file', path: require.resolve('./tengo/tpl/parse-bulk-count-matrix.plj.gz') },
+  'prerun': { type: 'from-file', path: require.resolve('./tengo/tpl/prerun.plj.gz') },
   'main': { type: 'from-file', path: require.resolve('./tengo/tpl/main.plj.gz') }
 }};

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "main";
+declare type TplName = "parse-bulk-count-matrix" | "prerun" | "main";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,4 +1,6 @@
 import { resolve } from 'node:path';
 export const Templates = {
+  'parse-bulk-count-matrix': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/parse-bulk-count-matrix.plj.gz') },
+  'prerun': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/prerun.plj.gz') },
   'main': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/main.plj.gz') }
 };

package/dist/tengo/lib/ds-bulk-count-matrix.lib.tengo

@@ -0,0 +1,67 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+render := import("@platforma-sdk/workflow-tengo:render")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+txt := import("@platforma-sdk/workflow-tengo:txt")
+parseHeaderTpl := assets.importTemplate("@platforma-open/milaboratories.samples-and-data.workflow:parse-bulk-count-matrix")
+
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: true,
+    
+    getSamples: func(dataset, importFile) {
+
+        result := {}
+        for groupId, importHandle in dataset.content.data {
+            if !importHandle {
+                ll.panic("File handle not set for group %v", groupId)
+            }
+            
+            headerContent := txt.head(importFile(importHandle), {lines: 1})
+
+            separator := "\t"
+            if dataset.content.xsvType == "tsv" {
+                separator = "\t"
+            } else if dataset.content.xsvType == "csv" {
+                separator = ","
+            } else {
+                ll.panic("Unsupported file extension: " + dataset.content.xsvType)
+            }
+            
+            samples := render.create(parseHeaderTpl, {
+                header: headerContent,
+                delimiter: separator
+            }).output("samples")
+            
+            result[groupId] = samples
+        }
+        return result
+    },
+
+    createDataset: func(blockId, sampleIdAxis, groupIdAxis, dataset, importFile) {
+        
+        extension :=  dataset.content.xsvType + (dataset.content.gzipped ? ".gz" : "")
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+        spec.axesSpec = [groupIdAxis]
+
+        data := pColumn.resourceMapBuilder(1)
+        for groupId, importHandle in dataset.content.data {
+            if !importHandle {
+                ll.panic("File handle not set for sample %v", groupId)
+            }
+            data.add([groupId], importFile(importHandle))
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/ds-cellranger-mtx.lib.tengo

@@ -0,0 +1,47 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+cellRangerMtxRoles := ["matrix.mtx", "features.tsv", "barcodes.tsv"]
+
+export {
+    isGrouped: false,
+
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        compression := dataset.content.gzipped ? "gz" : "none"
+
+        spec := util.datasetColumnSpecBase(blockId, dataset, undefined)
+        spec.domain["pl7.app/compression"] = compression
+
+        cellRangerFileRoleAxisSpec := {
+            type: "String",
+            name: "pl7.app/sc/cellRangerFileRole",
+            annotations: {
+                "pl7.app/label": "Role"
+            }
+        }
+
+        spec.axesSpec = [sampleIdAxis, cellRangerFileRoleAxisSpec]
+
+        data := pColumn.resourceMapBuilder(2)
+        for sampleId, fileGroup in dataset.content.data {
+            for role in cellRangerMtxRoles {
+                importHandle := fileGroup[role]
+                if !importHandle {
+                    ll.panic("File handle not set for %v in sample %v", role, sampleId)
+                }
+                data.add([sampleId, role], importFile(importHandle))
+            }
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}
+

package/dist/tengo/lib/ds-fasta.lib.tengo

@@ -0,0 +1,32 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: false,
+    
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        extension :=  dataset.content.gzipped ? "fasta.gz" : "fasta"
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+        spec.axesSpec = [sampleIdAxis]
+
+        data := pColumn.resourceMapBuilder(1)
+        for sampleId, importHandle in dataset.content.data {
+            if !importHandle {
+                ll.panic("File handle not set for sample %v", sampleId)
+            }
+            data.add([sampleId], importFile(importHandle))
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/ds-fastq.lib.tengo

@@ -0,0 +1,51 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+json := import("json")
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: false,
+    
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        extension :=  dataset.content.gzipped ? "fastq.gz" : "fastq"
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+
+        readIndexAxisSpec := {
+			type: "String",
+			name: "pl7.app/sequencing/readIndex",
+			annotations: {
+				"pl7.app/label": "Read Index"
+			},
+			domain: {
+
+
+
+
+				"pl7.app/readIndices": string(json.encode(dataset.content.readIndices))
+			}
+		}
+
+        spec.axesSpec = [sampleIdAxis, readIndexAxisSpec]
+
+        data := pColumn.resourceMapBuilder(2)
+        for sampleId, fileGroup in dataset.content.data {
+            for _, readIndex in dataset.content.readIndices {
+                importHandle := fileGroup[readIndex]
+                if !importHandle {
+                    ll.panic("File handle not set for %v in sample %v", readIndex, sampleId)
+                }
+                data.add([sampleId, readIndex], importFile(importHandle))
+            }
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/ds-multilane-fastq.lib.tengo

@@ -0,0 +1,61 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+json := import("json")
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: false,
+    
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        extension :=  dataset.content.gzipped ? "fastq.gz" : "fastq"
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+
+		laneAxisSpec := {
+			type: "String",
+			name: "pl7.app/sequencing/lane",
+			annotations: {
+				"pl7.app/label": "Lane"
+			}
+		}
+
+        readIndexAxisSpec := {
+			type: "String",
+			name: "pl7.app/sequencing/readIndex",
+			annotations: {
+				"pl7.app/label": "Read Index"
+			},
+			domain: {
+
+
+
+
+				"pl7.app/readIndices": string(json.encode(dataset.content.readIndices))
+			}
+		}
+
+        spec.axesSpec = [sampleIdAxis, laneAxisSpec, readIndexAxisSpec]
+
+        data := pColumn.resourceMapBuilder(3)
+        for sampleId, laneData in dataset.content.data {
+            for laneId, fileGroup in laneData {
+                for _, readIndex in dataset.content.readIndices {
+                    importHandle := fileGroup[readIndex]
+                    if !importHandle {
+                        ll.panic("File handle not set for %v, lane %v in sample %v", readIndex, laneId, sampleId)
+                    }
+                    data.add([sampleId, laneId, readIndex], importFile(importHandle))
+                }
+            }
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/ds-tagged-fastq.lib.tengo

@@ -0,0 +1,96 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+json := import("json")
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: false,
+    
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        extension :=  dataset.content.gzipped ? "fastq.gz" : "fastq"
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+
+        axesSpec := [sampleIdAxis]
+
+        for tag in dataset.content.tags {
+            axesSpec = append(axesSpec, {
+                type: "String",
+                name: "pl7.app/sequencing/tag",
+                domain: {
+                    "pl7.app/sequencing/tag": tag
+                },
+                annotations: {
+                    "pl7.app/label": tag + " tag"
+                }
+            })
+        }
+
+        if dataset.content.hasLanes {
+            laneAxisSpec := {
+                type: "String",
+                name: "pl7.app/sequencing/lane",
+                annotations: {
+                    "pl7.app/label": "Lane"
+                }
+            }
+
+            axesSpec = append(axesSpec, laneAxisSpec)
+        }
+
+        readIndexAxisSpec := {
+			type: "String",
+			name: "pl7.app/sequencing/readIndex",
+			annotations: {
+				"pl7.app/label": "Read Index"
+			},
+			domain: {
+
+
+
+
+				"pl7.app/readIndices": string(json.encode(dataset.content.readIndices))
+			}
+		}
+
+        axesSpec = append(axesSpec, readIndexAxisSpec)
+        spec.axesSpec = axesSpec
+
+        data := pColumn.resourceMapBuilder(len(axesSpec))
+        
+        for sampleId, records in dataset.content.data {
+            for record in records {
+                keyPrefix := [sampleId]
+                for tag in dataset.content.tags {
+                    tagValue := record.tags[tag]
+                    if !tagValue {
+                        ll.panic("No tag value for %v in sample %v", tag, sampleId)
+                    }
+                    keyPrefix = append(keyPrefix, tagValue)
+                }
+                if dataset.content.hasLanes {
+                    keyPrefix = append(keyPrefix, record.lane)
+                }
+                for readIndex in dataset.content.readIndices {
+                    importHandle := record.files[readIndex]
+                    key := maps.clone(keyPrefix)
+                    key = append(key, readIndex)
+                    if !importHandle {
+                        ll.panic("File handle not set for key %v", key)
+                    }
+                    data.add(key, importFile(importHandle))
+                }
+            }
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/ds-tagged-xsv.lib.tengo

@@ -0,0 +1,63 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: false,
+    
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        
+        extension :=  dataset.content.xsvType + (dataset.content.gzipped ? ".gz" : "")
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+        
+        axesSpec := [sampleIdAxis]
+
+         for tag in dataset.content.tags {
+            axesSpec = append(axesSpec, {
+                type: "String",
+                name: "pl7.app/sequencing/tag",
+                domain: {
+                    "pl7.app/sequencing/tag": tag
+                },
+                annotations: {
+                    "pl7.app/label": tag + " tag"
+                }
+            })
+        }
+
+        spec.axesSpec = axesSpec
+
+        data := pColumn.resourceMapBuilder(len(axesSpec))
+        
+        for sampleId, records in dataset.content.data {
+            for record in records {
+                key := [sampleId]
+                for tag in dataset.content.tags {
+                    tagValue := record.tags[tag]
+                    if !tagValue {
+                        ll.panic("No tag value for %v in sample %v", tag, sampleId)
+                    }
+                    key = append(key, tagValue)
+                }
+
+                importHandle := record.file
+                if !importHandle {
+                    ll.panic("File handle not set for key %v", key)
+                }
+                data.add(key, importFile(importHandle))
+            }
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/ds-xsv.lib.tengo

@@ -0,0 +1,33 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+pColumn := import("@platforma-sdk/workflow-tengo:pframes.pcolumn")
+
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+export {
+    isGrouped: false,
+    
+    createDataset: func(blockId, sampleIdAxis, dataset, importFile) {
+        
+        extension :=  dataset.content.xsvType + (dataset.content.gzipped ? ".gz" : "")
+        
+        spec := util.datasetColumnSpecBase(blockId, dataset, extension)
+        spec.axesSpec = [sampleIdAxis]
+
+        data := pColumn.resourceMapBuilder(1)
+        for sampleId, importHandle in dataset.content.data {
+            if !importHandle {
+                ll.panic("File handle not set for sample %v", sampleId)
+            }
+            data.add([sampleId], importFile(importHandle))
+        }
+
+        result := {}
+        result["dataset." + dataset.id] = {
+            spec: spec,
+            data: data.build()
+        }
+
+        return result
+    }
+}

package/dist/tengo/lib/metadata.lib.tengo

@@ -0,0 +1,63 @@
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+json := import("json")
+util := import("@platforma-open/milaboratories.samples-and-data.workflow:util")
+
+
+normalizeString := func(str) {
+	return str
+}
+
+
+
+
+
+
+
+
+
+
+createMetadata := func(args, blockId, sampleIdAxisSpec) {
+    columns := {}
+	for _, column in args.metadata {
+		columns["metadata." + column.id] = {
+			spec: {
+				kind: "PColumn",
+				name: "pl7.app/metadata",
+				valueType: column.valueType,
+				annotations: {
+					"pl7.app/label": column.label
+				},
+				domain: {
+					"pl7.app/columnId": column.global ? normalizeString(column.label) : column.id
+				},
+				axesSpec: [sampleIdAxisSpec]
+			},
+			data: util.createJsonPColumnData(column.data)
+		}
+	}
+
+
+
+
+
+	columns["metadata.sampleId"] = {
+		spec: {
+			kind: "PColumn",
+			name: "pl7.app/label",
+			valueType: "String",
+			annotations: {
+				"pl7.app/label": args.sampleLabelColumnLabel,
+				"pl7.app/isLabel": "true"
+			},
+			axesSpec: [sampleIdAxisSpec]
+		},
+		data: util.createJsonPColumnData(args.sampleLabels)
+	}
+
+    return columns
+}
+
+export {
+    createMetadata: createMetadata
+}

package/dist/tengo/lib/util.lib.tengo

@@ -0,0 +1,80 @@
+json := import("json")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+consts := import("@platforma-sdk/workflow-tengo:pframes.constants")
+
+RTYPE_P_COLUMN_DATA_JSON := consts.RTYPE_P_COLUMN_DATA_JSON
+
+mapToPValueData := func(map) {
+	data := {}
+	for key, value in map {
+		data[json.encode([key])] = value
+	}
+	result := {
+		keyLength: 1,
+		data: data
+	}
+	return result
+}
+
+createJsonPColumnData := func(data) {
+    return smart.createValueResource(RTYPE_P_COLUMN_DATA_JSON, json.encode(mapToPValueData(data)))
+}
+
+datasetColumnSpecBase := func(blockId, dataset, extension) {
+    return {
+        kind: "PColumn",
+        name: "pl7.app/sequencing/data",
+        domain: maps.clone({
+            "pl7.app/fileExtension": extension,
+            "pl7.app/block": blockId,
+            "pl7.app/dataset": dataset.id
+        }, { removeUndefs: true }),
+        valueType: "File",
+        annotations: {
+            "pl7.app/label": "Sequencing Data",
+            "pl7.app/hideDataFromUi": "true",
+            "pl7.app/axisKeys/0": string(json.encode(maps.getKeys(dataset.content.data))) // axis values for sampleIdAxis or groupIdAxis
+        }
+    }
+}
+
+sampleGroupsLinkerColumn := func(blockId, dataset, sampleIdAxis, groupIdAxis) {
+    data := {}
+    samples := []
+    for groupId, ss in dataset.content.sampleGroups {
+        for sampleId, sampleName in ss {
+            samples = samples + [sampleId]
+            data[json.encode([groupId, sampleId])] = sampleName
+        }
+    }
+    spec := {
+        kind: "PColumn",
+        name: "pl7.app/sequencing/data/sampleGroups",
+        domain: {
+            "pl7.app/block": blockId,
+            "pl7.app/dataset": dataset.id
+        },
+        valueType: "String",
+        axesSpec: [groupIdAxis, sampleIdAxis],
+        annotations: {
+            "pl7.app/label": "Sample Groups",
+            "pl7.app/isLinkedColumn": "true",
+            "pl7.app/hideDataFromUi": "true",
+            "pl7.app/axisKeys/0": string(json.encode(maps.getKeys(dataset.content.data))), // axis values for groupIdAxis
+            "pl7.app/axisKeys/1": string(json.encode(samples)) // axis values for sampleIdAxis
+        }
+    }
+
+    return {
+        spec: spec,
+        data: smart.createValueResource(RTYPE_P_COLUMN_DATA_JSON, json.encode(data))
+    }
+}
+
+
+export {
+    createJsonPColumnData: createJsonPColumnData,
+    datasetColumnSpecBase: datasetColumnSpecBase,
+    sampleGroupsLinkerColumn: sampleGroupsLinkerColumn
+}

package/package.json

@@ -1,16 +1,16 @@
 {
   "name": "@platforma-open/milaboratories.samples-and-data.workflow",
-  "version": "1.13.2",
+  "version": "2.0.0",
   "type": "module",
   "files": [
     "./dist/**/*"
   ],
   "description": "Tengo-based template",
   "dependencies": {
-    "@platforma-sdk/workflow-tengo": "^5.2.0"
+    "@platforma-sdk/workflow-tengo": "^5.4.5"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.1.13"
+    "@platforma-sdk/tengo-builder": "^2.3.2"
   },
   "scripts": {
     "build": "rm -rf dist/* && pl-tengo check && pl-tengo build",