@platforma-sdk/workflow-tengo 2.12.1 → 2.13.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "exec.monetization_to_json" | "pframes.aggregate" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.R.create-renv" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "pframes.export-pframe" | "exec.python.create-venv" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-map";
+declare type TplName = "exec.monetization_to_json" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.R.create-renv" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "exec.python.create-venv" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.export-pframe";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,6 +1,5 @@
 module.exports = { Templates: {
   'exec.monetization_to_json': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.monetization_to_json.plj.gz') },
-  'pframes.aggregate': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.aggregate.plj.gz') },
   'pframes.export-single-pcolumn': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
   'pframes.import-dir': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.import-dir.plj.gz') },
   'pframes.map-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.map-pframe.plj.gz') },
@@ -9,9 +8,10 @@ module.exports = { Templates: {
   'exec.R.create-renv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.R.create-renv.plj.gz') },
   'exec.python.install-deps': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.install-deps.plj.gz') },
   'pframes.export-multiple-pcolumns': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
-  'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') },
   'exec.python.create-venv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.create-venv.plj.gz') },
   'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
   'pframes.xsv-export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
-  'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') }
+  'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') },
+  'pframes.process-pcolumn-data': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
+  'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') }
 }};

package/dist/index.mjs

@@ -1,7 +1,6 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'exec.monetization_to_json': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.monetization_to_json.plj.gz') },
-  'pframes.aggregate': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.aggregate.plj.gz') },
   'pframes.export-single-pcolumn': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
   'pframes.import-dir': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.import-dir.plj.gz') },
   'pframes.map-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.map-pframe.plj.gz') },
@@ -10,9 +9,10 @@ export const Templates = {
   'exec.R.create-renv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.R.create-renv.plj.gz') },
   'exec.python.install-deps': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.install-deps.plj.gz') },
   'pframes.export-multiple-pcolumns': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
-  'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') },
   'exec.python.create-venv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.create-venv.plj.gz') },
   'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
   'pframes.xsv-export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
-  'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') }
+  'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') },
+  'pframes.process-pcolumn-data': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
+  'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') }
 };

package/dist/tengo/lib/pframes.builder.lib.tengo

@@ -0,0 +1,35 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+
+
+/**
+ * Creates a builder for p-frame resource.
+ */
+pFrameBuilder := func() {
+	r := smart.structBuilder(pConstants.RTYPE_P_FRAME)
+
+	self := undefined
+	self = ll.toStrict(oop.inherit(r, {
+
+		add: func(column, spec, data) {
+			if (!is_undefined(spec)) {
+				r.createInputField(column + ".spec").setRefOrJson(spec)
+			}
+			if (!is_undefined(data)) {
+				r.createInputField(column + ".data").setRefOrJson(data)
+			}
+			return self
+		},
+
+		build: func() {
+			return r.lockAndBuild()
+		}
+	}))
+	return self
+}
+
+export ll.toStrict({
+	pFrameBuilder: pFrameBuilder
+})

package/dist/tengo/lib/pframes.constants.lib.tengo

@@ -17,6 +17,18 @@ VALUE_FIELD_NAME := "__value__"
  */
 KEY_FIELD_NAME := "__key__"
 
+/**
+ * Name of the field under which the aggregation axes specification is passed
+ * to the iteration body template.
+ */
+AGGREGATION_AXES_SPEC_FIELD_NAME := "__aggregation_axes_spec__"
+
+/**
+ * Name of the field under which the aggregation axes names are passed
+ * to the iteration body template.
+ */
+AGGREGATION_AXES_NAMES_FIELD_NAME := "__aggregation_axes_names__"
+
 RTYPE_P_FRAME :=  { Name: "PFrame", Version: "1" }
 
 P_COLUMN_DATA_TYPE_PREFIX := "PColumnData/"
@@ -37,6 +49,8 @@ export ll.toStrict({
 	KIND_P_COLUMN                                 : KIND_P_COLUMN,
 	KEY_FIELD_NAME                                : KEY_FIELD_NAME,
 	VALUE_FIELD_NAME                              : VALUE_FIELD_NAME,
+	AGGREGATION_AXES_SPEC_FIELD_NAME             : AGGREGATION_AXES_SPEC_FIELD_NAME,
+	AGGREGATION_AXES_NAMES_FIELD_NAME            : AGGREGATION_AXES_NAMES_FIELD_NAME,
 	P_COLUMN_DATA_TYPE_PREFIX                     : P_COLUMN_DATA_TYPE_PREFIX,
 	P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX   : P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX,
 

package/dist/tengo/lib/pframes.lib.tengo

@@ -3,12 +3,19 @@
  */
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
-oop := import("@platforma-sdk/workflow-tengo:oop")
+maps := import("@platforma-sdk/workflow-tengo:maps")
 smart := import("@platforma-sdk/workflow-tengo:smart")
+slices := import("@platforma-sdk/workflow-tengo:slices")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 render := import("@platforma-sdk/workflow-tengo:render")
+builder := import("@platforma-sdk/workflow-tengo:pframes.builder")
+pUtil := import("@platforma-sdk/workflow-tengo:pframes.util")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+validation := import("@platforma-sdk/workflow-tengo:validation")
 constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 
+llProcessTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.process-pcolumn-data")
+
 /**
  * Given a reference to the p-frame, converts all inner resource files into resources that can be
  * downloaded from the UI side.
@@ -25,41 +32,500 @@ exportColumnData := func(data) {
 }
 
 /**
- * Creates a builder for p-frame resource.
+ * Creates a builder for a resource map p-column's data.
  */
-pFrameBuilder := func() {
-	r := smart.structBuilder(constants.RTYPE_P_FRAME)
+pColumnResourceMapDataBuilder := func(keyLength) {
+	ll.panic("pframes.pColumnResourceMapDataBuilder is not supported any more; use pcolumn.resourceMapBuilder")
+}
+
+/**
+ * Constructs a mapping or aggregation pipeline for a given input PColumn and returns a specialized
+ * output object for accessing the processing results.
+ *
+ * === Body Template Contract ===
+ *
+ *   Inputs:
+ *     "__key__"   - When passKey is true, contains the JSON representation of the group/element key:
+ *                     - Without aggregation: represents a single key
+ *                     - With aggregation: represents all keys in positions not specified by indices
+ *                   Not provided if passKey is false or omitted from ops
+ *     "__aggregation_axes_spec__"
+ *                 - When passAggregationAxesSpec is true, contains the axes specification for aggregation
+ *                     Can be used to adapt body behavior when optional aggregation axes are unmatched
+ *                   Not provided if passAggregationAxesSpec is false or omitted from ops
+ *     "__aggregation_axes_names__"
+ *                 - When passAggregationAxesNames is true, contains just the names of aggregation axes
+ *                     as a simple array of strings, without domains, annotations, etc.
+ *                   Not provided if passAggregationAxesNames is false or omitted from ops
+ *     "__value__" - Content depends on aggregation setting:
+ *                     - Without aggregation: single value (only for ResourceMap PColumns)
+ *                     - With aggregation: "PColumnData/*" resource containing all records in current group
+ *     ...extra    - Additional values/references passed via the options.extra parameter
+ *                   are forwarded to the template unchanged
+ *
+ * === Parameters ===
+ *
+ * @param input         Primary input column data and specifications
+ *
+ *                      Required structure:
+ * 							{
+ * 							 	// PColumn specifications (must be explicit, not a reference)
+ * 							 	spec: <PColumnSpec>,
+ *
+ * 							 	// Reference to the data resource
+ * 							 	data: <resource ref>
+ *							}
+ *
+ * @param bodyTpl       Template executed for each iteration (see contract above)
+ * @param outputs 		Array of output configurations to collect from rendered templates
+ *
+ *                  Common fields for all types:
+ *                     {
+ *                       name:  string     // Output name
+ *                       path?: string[]   // [Optional] Override path inside body template result
+ *                                            First element interpreted as output name
+ *                                            Defaults to [name] if omitted
+ *                     }
+ *
+ *                  Available types:
+ *                    | {
+ *                        type: "Resource",     // Single resource output
+ *                        spec: PColumnSpec     // Result column spec (must have zero axes specs,
+ *                                              // input axes specs added automatically)
+ *                      }
+ *                    | {
+ *                        type: "ResourceMap",  // ResourceMap output
+ *                        spec: PColumnSpec     // Result column spec (list only axes specs
+ *                                              // produced by body template, input axes added automatically)
+ *                      }
+ *                    | {
+ *                        type: "BinaryPartitioned" | "JsonPartitioned",
+ *                                              // Partitioned Json/Binary PColumn data
+ *                        spec: PColumnSpec     // Result column spec (list only axes specs
+ *                                              // produced by body template, input axes added automatically)
+ *                        partitionKeyLength: number
+ *                                              // Expected partition length per operation
+ *                      }
+ *                    | {
+ *                        type: "Xsv",          // CSV/TSV formatted text file for xsv conversion
+ *						  xsvType: "tsv"|"csv", // Expected file format
+ *                        settings: object      // Xsv conversion specification
+ *                      }
+ *
+ * @param opts 			Optional additional settings
+ *
+ *                    {
+ *                      traceSteps?: array      // Array of trace steps to add to output specs
+ *                                              // Each step should be an object with type, id, importance and label
+ *
+ *                      aggregate?: AxisMatcher[],
+ *                                              // If specified, selected axes are combined and
+ *                                              // passed to each iteration as a group. These axes
+ *                                              // won't be automatically added to output specs
+ *                                              // Examples:
+ *                                              //   - "pl7.app/sequencing/readIndex"
+ *                                              //   - { name: "pl7.app/sequencing/readIndex" }
+ *                                              //   - { type: "String", name: "pl7.app/sequencing/lane" }
+ *                                              //   - { type: "String", name: "pl7.app/sequencing/lane", optional: true }
+ *                                              //   - { type: "String", name: "pl7.app/rna-seq/geneId", domain: { "pl7.app/species": "homo-sapiens" } }
+ *                                              // With optional: true, matcher won't error if unmatched
+ *                                              // Use passAggregationAxesSpec for custom handling in body template
+ *
+ *                      passKey?: boolean       // Pass __key__ to body template if true
+ *
+ *                      passAggregationAxesSpec?: boolean
+ *                                              // Pass full aggregation axes specs to body template
+ *                                              // via __aggregation_axes_spec__ input
+ *
+ *                      passAggregationAxesNames?: boolean
+ *                                              // Pass array of aggregation axes names to body template
+ *                                              // via __aggregation_axes_names__ input
+ *
+ *                      extra?: Map<string, Ref>,
+ *                                              // Additional parameters passed to each body invocation
+ *                                              // Keys match template input names
+ *
+ *                      isEphemeral?: boolean   // Whether to render mapping template in ephemeral mode
+ *                                              // Defaults to false if omitted, template will be rendered
+ *                                              // as pure resource
+ *                    }
+ *
+ * === Return ===
+ *
+ * @returns Object with methods for retrieving processing results
+ *          See source code for method documentation and signatures
+ */
+processColumn := func(input, bodyTpl, outputs, ...opts) {
+	if len(opts) == 0 {
+		opts = {}
+	} else if len(opts) == 1 {
+		opts = opts[0]
+	} else {
+		ll.panic("too many arguments")
+	}
+
+	if is_undefined(input.spec) {
+		ll.panic("no input spec provided")
+	}
+
+	if !smart.isResource(input.data) {
+		ll.panic("input data should be a resource")
+	}
+
+	validation.assertType(input.spec, pSpec.P_COLUMN_SPEC_SCHEMA)
+
+	if is_undefined(opts.passKey) {
+		opts.passKey = false
+	}
+
+	if is_undefined(opts.passAggregationAxesSpec) {
+		opts.passAggregationAxesSpec = false
+	}
+
+	if is_undefined(opts.passAggregationAxesNames) {
+		opts.passAggregationAxesNames = false
+	}
+
+	if is_undefined(opts.isEphemeral) {
+		opts.isEphemeral = false
+	}
+
+	if (opts.passAggregationAxesSpec || opts.passAggregationAxesNames) && is_undefined(opts.aggregate) {
+		ll.panic("can't use passAggregationAxesSpec or passAggregationAxesNames without aggregate")
+	}
+
+	// Initialize trace if traceSteps provided
+	trace := undefined
+	if !is_undefined(opts.traceSteps) {
+		trace = pSpec.makeTrace(input.spec, opts.traceSteps...)
+	}
+
+	// JSON params passed as a single resource to the template
+	processTemplateParams := {
+		eph: opts.isEphemeral,
+		passKey: opts.passKey
+	}
+
+	extra := opts.extra
+	if is_undefined(extra) {
+		extra = {}
+	}
+
+	iterationAxesSpec := input.spec.axesSpec
+	processTemplateParams.expectedKeyLength = len(input.spec.axesSpec)
+	if !is_undefined(opts.aggregate) {
+		aggregationIndices := pSpec.matchAxes(input.spec.axesSpec, opts.aggregate)
+		processTemplateParams.aggregationIndices = aggregationIndices
+
+		// calculate iteration axes spec
+		groupAxesIndices := pUtil.calculateGroupAxesIndices(len(input.spec.axesSpec), aggregationIndices)
+		iterationAxesSpec = []
+		for i in groupAxesIndices {
+			iterationAxesSpec = append(iterationAxesSpec, input.spec.axesSpec[i])
+		}
+
+		if opts.passAggregationAxesSpec {
+			aggregationAxesSpec := []
+			for i in aggregationIndices {
+				aggregationAxesSpec = append(aggregationAxesSpec, input.spec.axesSpec[i])
+			}
+
+			extra[constants.AGGREGATION_AXES_SPEC_FIELD_NAME] = smart.createJsonResource(aggregationAxesSpec)
+		}
+
+		if opts.passAggregationAxesNames {
+			aggregationAxesNames := []
+			for i in aggregationIndices {
+				aggregationAxesNames = append(aggregationAxesNames, input.spec.axesSpec[i].name)
+			}
+
+			extra[constants.AGGREGATION_AXES_NAMES_FIELD_NAME] = smart.createJsonResource(aggregationAxesNames)
+		}
+	}
+
+	processedOutputs := []
+	outputSpecs := {}
+	outputsMap := {}
+	for output in outputs {
+		// map output name to output settings provided in input
+		outputsMap[output.name] = output
+		if output.type == "Resource" || output.type == "BinaryPartitioned" || output.type == "JsonPartitioned" {
+			spec := maps.merge(output.spec, { axesSpec: iterationAxesSpec + slices.normalize(output.spec.axesSpec) })
+			if !is_undefined(trace) {
+				spec = trace.inject(spec)
+			}
+			outputSpecs[output.name] = spec
+			processedOutputs = append(processedOutputs, maps.merge(output, {spec: undefined })) // removing spec field
+		} else if output.type == "ResourceMap" {
+			processedOutputs = append(processedOutputs, maps.merge(output,
+				{ keyLength: len(output.spec.axesSpec), spec: undefined }))
+			spec := maps.merge(output.spec, { axesSpec: iterationAxesSpec + slices.normalize(output.spec.axesSpec) })
+			if !is_undefined(trace) {
+				spec = trace.inject(spec)
+			}
+			outputSpecs[output.name] = spec
+		} else if output.type == "Xsv" {
+			decomposition := pUtil.decomposePfconvImportCfg(output.settings, {
+				additionalAxesSpec: iterationAxesSpec
+			})
+			purifiedPfconvCfg := decomposition.purifiedCfg
+			columnsSpec := decomposition.columnsSpec
+			if !is_undefined(trace) {
+				for columnName, spec in columnsSpec {
+					columnsSpec[columnName] = trace.inject(spec)
+				}
+			}
+			outputSpecs[output.name] = columnsSpec
+			processedOutputs = append(processedOutputs, maps.merge(output,
+				{
+					settings: purifiedPfconvCfg,
+					flattenWithDelimiter: "/"
+				}))
+		} else {
+			ll.panic("unknown output type: " + output.type)
+		}
+	}
+	processTemplateParams.outputs = processedOutputs
+
+	renderInputs := {
+		params: smart.createJsonResource(processTemplateParams),
+		body: bodyTpl,
+		data: input.data
+	}
+
+	for k, v in extra {
+		renderInputs["__extra_" + k] = v
+	}
+
+	renderResult := render.createEphemeral(llProcessTpl, renderInputs)
 
 	self := undefined
-	self = ll.toStrict(oop.inherit(r, {
+	self = ll.toStrict({
+		renderResult: renderResult,
+
+		/**
+		 * Returns output spec for the given output name and optional column name
+		 *
+		 * For normal outputs (Resource, ResourceMap, BinaryPartitioned, JsonPartitioned),
+		 * only the output name should be provided.
+		 * For Xsv outputs, both output name and column name should be provided.
+		 *
+		 * @param name - Name of the output
+		 * @param column - Optional. For Xsv outputs, name of the column
+		 * @returns Output spec
+		 */
+		outputSpec: func(name, ...column) {
+			output := outputsMap[name]
+			if is_undefined(output) {
+				ll.panic("unknown output: " + name)
+			}
+
+			spec := outputSpecs[name]
+			if output.type == "Xsv" {
+				if len(column) != 1 {
+					ll.panic("column name must be provided for Xsv output")
+				}
+				columnName := column[0]
+				spec = spec[columnName]
+				if is_undefined(spec) {
+					ll.panic("unknown column: " + columnName)
+				}
+			}
+
+			if len(column) > 0 {
+				ll.panic("column name should not be provided for non-Xsv output")
+			}
+			return spec
+		},
+
+		/**
+		 * Returns data resource for the given output name and optional column name
+		 *
+		 * For normal outputs (Resource, ResourceMap, BinaryPartitioned, JsonPartitioned),
+		 * only the output name should be provided.
+		 * For Xsv outputs, both output name and column name should be provided.
+		 *
+		 * @param name - Name of the output
+		 * @param column - Optional. For Xsv outputs, name of the column
+		 * @returns data resource
+		 */
+		outputData: func(name, ...column) {
+			output := outputsMap[name]
+			if is_undefined(output) {
+				ll.panic("unknown output: " + name)
+			}
+
+			if output.type == "Xsv" {
+				if len(column) != 1 {
+					ll.panic("column name must be provided for Xsv output")
+				}
+				return renderResult.output(name + "/" + column[0])
+			}
+
+			if len(column) > 0 {
+				ll.panic("column name should not be provided for non-Xsv output")
+			}
+			return renderResult.output(name)
+		},
+
+		/**
+		 * Returns both data resource and spec for the given output name and optional column name
+		 *
+		 * For normal outputs (Resource, ResourceMap, BinaryPartitioned, JsonPartitioned),
+		 * only the output name should be provided.
+		 * For Xsv outputs, both output name and column name should be provided.
+		 *
+		 * @param name - Name of the output
+		 * @param column - Optional. For Xsv outputs, name of the column
+		 * @returns {data: Resource, spec: PColumnSpec} Object containing both data resource and spec
+		 */
+		output: func(name, ...column) {
+			output := outputsMap[name]
+			if is_undefined(output) {
+				ll.panic("unknown output: " + name)
+			}
+
+			if output.type == "Xsv" {
+				if len(column) != 1 {
+					ll.panic("column name must be provided for Xsv output")
+				}
+				columnName := column[0]
+				spec := outputSpecs[name][columnName]
+				if is_undefined(spec) {
+					ll.panic("unknown column: " + columnName)
+				}
+				return {
+					data: renderResult.output(name + "/" + columnName),
+					spec: spec
+				}
+			}
+
+			if len(column) > 0 {
+				ll.panic("column name should not be provided for non-Xsv output")
+			}
+			return {
+				data: renderResult.output(name),
+				spec: outputSpecs[name]
+			}
+		},
+
+		/**
+		 * Returns a PFrame containing all columns from an Xsv output
+		 *
+		 * @param name - Name of the Xsv output
+		 * @returns PFrame containing all columns from the output
+		 */
+		xsvOutputFrame: func(name) {
+			output := outputsMap[name]
+			if is_undefined(output) {
+				ll.panic("unknown output: " + name)
+			}
+
+			if output.type != "Xsv" {
+				ll.panic("outputFrame can only be used with Xsv outputs")
+			}
+
+			builder := builder.pFrameBuilder()
+			for columnName, spec in outputSpecs[name] {
+				builder.add(columnName, spec, renderResult.output(name + "/" + columnName))
+			}
+			return builder.build()
+		},
 
-		add: func(column, spec, data) {
-			if (!is_undefined(spec)) {
-				r.createInputField(column + ".spec").setRefOrJson(spec)
+		/**
+		 * Returns a PFrame containing all results, flattening Xsv outputs using the provided separator
+		 *
+		 * @param ...xsvNameSeparator - Optional separator for joining Xsv output and column names, defaults to "."
+		 * @returns PFrame containing all results
+		 */
+		allOutputsFrame: func(...xsvNameSeparator) {
+			sep := "."
+			if len(xsvNameSeparator) > 0 {
+				sep = xsvNameSeparator[0]
 			}
-			if (!is_undefined(data)) {
-				r.createInputField(column + ".data").setRefOrJson(data)
+
+			builder := builder.pFrameBuilder()
+
+			for name, output in outputsMap {
+				if output.type == "Xsv" {
+					// For Xsv outputs, add each column with name prefixed
+					for columnName, spec in outputSpecs[name] {
+						builder.add(
+							name + sep + columnName,
+							spec,
+							renderResult.output(name + "/" + columnName)
+						)
+					}
+				} else {
+					// For non-Xsv outputs, add directly
+					builder.add(
+						name,
+						outputSpecs[name],
+						renderResult.output(name)
+					)
+				}
 			}
-			return self
+
+			return builder.build()
 		},
 
-		build: func() {
-			return r.lockAndBuild()
+		/**
+		 * Returns sorted list of all available output names
+		 *
+		 * @returns Array of output names
+		 */
+		listOutputs: func() {
+			outputs := []
+			for name, _ in outputsMap {
+				outputs = append(outputs, name)
+			}
+			return slices.quickSort(outputs)
+		},
+
+		/**
+		 * Checks if given output is of Xsv type
+		 *
+		 * @param name - Output name to check
+		 * @returns true if output is Xsv, false otherwise
+		 */
+		isXsvOutput: func(name) {
+			output := outputsMap[name]
+			if is_undefined(output) {
+				ll.panic("unknown output: " + name)
+			}
+			return output.type == "Xsv"
+		},
+
+		/**
+		 * Returns sorted list of column names for given Xsv output
+		 *
+		 * @param name - Xsv output name
+		 * @returns Array of column names
+		 */
+		listXsvColumns: func(name) {
+			output := outputsMap[name]
+			if is_undefined(output) {
+				ll.panic("unknown output: " + name)
+			}
+			if output.type != "Xsv" {
+				ll.panic("output is not Xsv: " + name)
+			}
+
+			columns := []
+			for columnName, _ in outputSpecs[name] {
+				columns = append(columns, columnName)
+			}
+			return slices.quickSort(columns)
 		}
-	}))
-	return self
-}
+	})
 
-/**
- * Creates a builder for a resource map p-column's data.
- */
-pColumnResourceMapDataBuilder := func(keyLength) {
-	ll.panic("pframes.pColumnResourceMapDataBuilder is not supported any more; use pcolumn.resourceMapBuilder")
+	return self
 }
 
 export ll.toStrict({
+	processColumn: processColumn,
 	exportFrame: exportFrame,
-	pFrameBuilder: pFrameBuilder,
+	pFrameBuilder: builder.pFrameBuilder,
 	exportColumnData: exportColumnData,
 	pColumnResourceMapDataBuilder: pColumnResourceMapDataBuilder
 })

package/dist/tengo/lib/pframes.ll.lib.tengo

@@ -7,62 +7,68 @@ assets := import("@platforma-sdk/workflow-tengo:assets")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 render := import("@platforma-sdk/workflow-tengo:render")
 
-llAggTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.aggregate")
+llProcessTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.process-pcolumn-data")
 
 /**
- * Instantiate an aggregation pipeline for the given inputs, and returns rendered ephemeral template.
- * Use r.output("output_name") method to get corresponding result, where r is an object returned by
+ * Instantiates an aggregation pipeline for the given inputs and returns rendered ephemeral template.
+ * Use result.output("outputName") to get corresponding output, where result is the object returned by
  * this method.
  *
  * Contract for the body template:
- *
  *   Inputs:
- *     "__key__"   - will be set to the json representation of the group key (i.e. keys in all
- *                   positions not specified in the indices parameter)
- *     "__value__" - "PColumnData/ResourceMap" resource with all the records in the current group
- *     ...extra    - all values or references passed to optional last parameter of this method
- *                   will be forwarded to the template as is
- *
- *  Outputs:
- *    array of output names to collect from aggregation operations is specified as a parameter
- *    of this operation
- *
- * @param data    data resource (currently only PColumnData/ResourceMap is supported)
- * @param indices indices to aggregate over (these indices will be stripped from the input map,
- *                meaning of this parameter is the opposite of GROUP BY from SQL)
- * @param bodyTpl reference to the template implementing particular aggregation (see contract above)
- * @param outputs array of output settings to to collect from rendered templates
- *
- *                  Common fields for all subtypes:
- *                     {
- *                       name:  string     - name of the output
- *                       path?: string[]   - [optional] overrides the path inside the body template
- *                                           result, first name is interpreted as output name. If
- *                                           not specified path equals to `[name]`.
- *                     }
+ *     "__key__": string     - JSON representation of the group key (i.e. keys in all positions not
+ *                             specified in the indices parameter). Only provided if passKey is true.
+ *     "__value__": Resource - PColumnData/ResourceMap resource containing all records in current group
+ *     ...extra              - Additional values/references passed via the extra parameter are forwarded
+ *                             to the template unchanged
  *
- *                  Variants:
- *                    | { type: "Resource" } // - assume normal singular resource is produced by
- *                                                this output
- *                    | {
- *                        type: "ResourceMap",
- *                        keyLength: number // - expected key length of result from each operation
- *                      } // - assume each record is ResourceMap
- *                    | {
- *                        type: "BinaryPartitioned" | "JsonPartitioned",
- *                        partitionKeyLength: number // - expected partition length of result from
- *                                                        each operation
- *                      } // - assume each record is a partitionded Json or Binary PColumn data itself
+ * @param data: Resource                - Data resource (currently only PColumnData/ResourceMap supported)
+ * @param aggregationIndices: number[]|undefined
+ *                                      - Indices to aggregate over. These indices will be stripped from
+ *                                       the input map (opposite of SQL GROUP BY). If undefined, works as
+ *                                       "map" and passes each input value as raw resource.
+ * @param bodyTpl: Template            - Template implementing the aggregation logic
+ * @param outputs: OutputSpec[]        - Array of output specifications to collect from rendered templates
+ *                                      Common fields for all output types:
+ *                                        {
+ *                                          name: string      - Name of the output
+ *                                          path?: string[]   - Optional. Overrides path inside body template
+ *                                                             result. First element is output name. Defaults
+ *                                                             to [name] if not specified.
+ *                                        }
+ *                                      Output type variants:
+ *                                        | { type: "Resource" }  - Normal singular resource
+ *                                        | {                     - ResourceMap output
+ *                                            type: "ResourceMap",
+ *                                            keyLength: number   - Expected key length per operation
+ *                                          }
+ *                                        | {                     - Partitioned data output
+ *                                            type: "BinaryPartitioned"|"JsonPartitioned",
+ *                                            partitionKeyLength: number - Expected partition length per op
+ *                                          }
+ *                                        | {                     - CSV/TSV text file output
+ *                                            type: "Xsv",
+ *                                            xsvType: "tsv"|"csv",
+ *                                            flattenWithDelimiter?: string,
+ *                                                                - If present, flattens output by adding columns
+ *                                                                  one by one using the specified delimiter:
+ *                                                                  "{outputName}{flattenWithDelimiter}{columnId}"
+ *                                            settings: object    - Xsv conversion specification
+ *                                          }
+ * @param isEph: boolean               - If true, body template will be rendered as ephemeral
+ * @param passKey: boolean             - If true, body template will receive __key__ input
+ * @param extra: object                - Optional map of parameters to pass to the template
  *
- * @param isEph   if true, body template will be rendered as ephemeral
- * @param extra   optional map of parameters to pass to the template
+ * @returns RenderResult              - Rendered template result with output() method for accessing outputs
+ * @panics                           - If extra parameter count is invalid
  */
-aggregate := func(data, indices, bodyTpl, outputs, isEph, ...extra) {
+aggregateV2 := func(data, aggregationIndices, bodyTpl, outputs, isEph, passKey, ...extra) {
 	renderInputs := {
 		params: smart.createJsonResource({
-			indices: indices,
+			aggregationIndices: aggregationIndices,
 			eph: isEph,
-			outputs: outputs
+			outputs: outputs,
+			passKey: passKey
 		}),
 		body: bodyTpl,
 		data: data
@@ -76,10 +82,18 @@ aggregate := func(data, indices, bodyTpl, outputs, isEph, ...extra) {
 		ll.panic("Wrong number of parameters for aggregate method.")
 	}
 
-	renderResult := render.createEphemeral(llAggTpl, renderInputs)
-	return renderResult;
+	renderResult := render.createEphemeral(llProcessTpl, renderInputs)
+	return renderResult
+}
+
+/*
+ * Read docs for aggregateV2
+ */
+aggregate := func(data, aggregationIndices, bodyTpl, outputs, isEph, ...extra) {
+	return aggregateV2(data, aggregationIndices, bodyTpl, outputs, isEph, true, extra...)
 }
 
 export ll.toStrict({
-	aggregate: aggregate
+	aggregate: aggregate,
+	aggregateV2: aggregateV2
 })

package/dist/tengo/lib/pframes.spec.lib.tengo

@@ -68,6 +68,133 @@ A_LABEL := "pl7.app/label"
  */
 A_TRACE := "pl7.app/trace"
 
+//
+// Format methods
+//
+
+normalizeColumnSpec := func(spec) {
+	if is_undefined(spec.axesSpec) {
+		spec.axesSpec = []
+	}
+	validation.assertType(spec, P_COLUMN_SPEC_SCHEMA)
+	return spec
+}
+
+//
+// Matchers
+//
+
+_P_MATCHER_SCHEMA := {
+	`__options__,closed`: ``,
+	`type,?`: `string,regex=Int|Long|Float|Double|String`,
+	`name`: `string`,
+	`domain,?`: { any: `string` },
+	`optional,?`: `boolean`
+}
+
+_normalizeMatcher := func(matcher) {
+	if is_string(matcher) {
+		return { name: matcher }
+	} else {
+		validation.assertType(matcher, _P_MATCHER_SCHEMA)
+		return matcher
+	}
+}
+
+_checkAxisMatch := func(axisSpec, matcher) {
+	if axisSpec.name != matcher.name {
+		return false
+	}
+	if !is_undefined(matcher.type) && axisSpec.type != matcher.type {
+		return false
+	}
+	if !is_undefined(matcher.domain) {
+		for domain, domainValue in matcher.domain {
+			if is_undefined(axisSpec.domain) || axisSpec.domain[domain] != domainValue {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+/**
+ * For a given array of axes specs returns the index of matching axis:
+ *   - if multiple matches are found - panics
+ *   - if axis not found and matcher is not optional - panics
+ */
+matchAxis := func(axesSpec, matcher) {
+	validation.assertType(axesSpec, [ P_AXIS_SPEC_SCHEMA ])
+	matcher = _normalizeMatcher(matcher)
+	found := -1
+	for idx, axisSpec in axesSpec {
+		if !_checkAxisMatch(axisSpec, matcher) {
+			continue
+		}
+		if found != -1 {
+			ll.panic("multiple matches found for %v in %v", matcher, axesSpec)
+		}
+		found = idx
+	}
+	if found == -1 && !matcher.optional {
+		ll.panic("match not found for %v in %v", matcher, axesSpec)
+	}
+	return found
+}
+
+/**
+ * Matches axes specs against matchers and returns array of matched indices.
+ *
+ * Example matchers:
+ *   - "pl7.app/sequencing/readIndex"
+ *   - { name: "pl7.app/sequencing/readIndex" }
+ *   - { type: "String", name: "pl7.app/sequencing/lane" }
+ *   - { type: "String", name: "pl7.app/sequencing/lane", optional: true }
+ *   - { type: "String", name: "pl7.app/rna-seq/geneId", domain: { "pl7.app/species": "homo-sapiens" } }
+ *
+ * @param axesSpec: AxisSpec[] Array of axis specifications to match against
+ * @param matchers: (string|AxisMatcher)[] Array of matchers, each can be either:
+ *   - string: Simple name matcher e.g. "pl7.app/sequencing/readIndex"
+ *   - AxisMatcher: Object with matching criteria:
+ *     - name: string - Axis name to match
+ *     - type?: string - Optional axis type to match
+ *     - domain?: Record<string,string> - Optional domain criteria
+ *     - optional?: boolean - If true, won't panic when no match found
+ * @param opts?: object Optional parameters:
+ *   - keepUnmatched?: boolean - Whether to include unmatched indices (-1) in result
+ * @returns number[] Array of matched indices in same order as matchers. Contains -1 for
+ *          unmatched optional matchers if keepUnmatched is true.
+ * @panics
+ *   - If multiple axes match a single matcher
+ *   - If no match found for non-optional matcher
+ *   - If multiple matchers match the same axis
+ */
+matchAxes := func(axesSpec, matchers, ...opts) {
+	keepUnmatched := false
+	if len(opts) == 1 {
+		keepUnmatched = opts[0].keepUnmatched
+	} else if len(opts) != 0 {
+		ll.panic("wrong number of parameters for matchAxes")
+	}
+
+	result := []
+	resultSet := {}
+	for matcher in matchers {
+		idx := matchAxis(axesSpec, matcher)
+		if idx != -1 {
+			idxS := string(idx)
+			if resultSet[idxS] {
+				ll.panic("duplicate match detecterd for %v in %v (idx = %v)", matchers, axesSpec, idx)
+			}
+			resultSet[idxS] = true
+		}
+		if idx != -1 || keepUnmatched {
+			result = append(result, idx)
+		}
+	}
+	return result
+}
+
 //
 // Utility methods
 //
@@ -123,6 +250,9 @@ makeTrace := func(input, ...steps) {
 }
 
 export ll.toStrict({
+	P_AXIS_SPEC_SCHEMA: P_AXIS_SPEC_SCHEMA,
+	P_COLUMN_SPEC_SCHEMA: P_COLUMN_SPEC_SCHEMA,
+
 	KIND_P_COLUMN: KIND_P_COLUMN,
 
 	N_LABEL: N_LABEL,
@@ -132,5 +262,10 @@ export ll.toStrict({
 	A_LABEL: A_LABEL,
 	A_TRACE: A_TRACE,
 
+	normalizeColumnSpec: normalizeColumnSpec,
+
+	matchAxis: matchAxis,
+	matchAxes: matchAxes,
+
 	makeTrace: makeTrace
 })

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -1,6 +1,7 @@
 ll := import("@platforma-sdk/workflow-tengo:ll")
 validation := import("@platforma-sdk/workflow-tengo:validation")
 objects := import("@platforma-sdk/workflow-tengo:objects")
+slices := import("@platforma-sdk/workflow-tengo:slices")
 
 _PRE_PROCESS_STEP_SCHEMA := [`or`,
 	{
@@ -317,11 +318,46 @@ decomposePfconvImportCfg := func(cfg, ...ops) {
 	}
 }
 
+EMPTY_JSON_KEY := "[]" // string(json.encode([]))
+
+/**
+ * Calculates group indices from aggregation indices by removing aggregation indices from the full set of indices.
+ * If no aggregation indices are provided, returns all indices.
+ *
+ * @param numberOfAxes: number - Total number of axes in the input
+ * @param aggregationIndices: number[]|undefined - Indices to aggregate over. If undefined, returns all indices.
+ *
+ * @returns number[] - Array of remaining indices after removing aggregation indices, in ascending order.
+ *                     If aggregationIndices is undefined, returns array [0..numberOfAxes-1]
+ */
+calculateGroupAxesIndices := func(numberOfAxes, aggregationIndices) {
+	groupIndices := []
+	for i:=0; i<numberOfAxes; i++ {
+		groupIndices = append(groupIndices, i)
+	}
+
+	// groupIndices at this point is [0, 1, 2, ... numberOfAxes-1]
+
+	if is_undefined(aggregationIndices) {
+		return groupIndices
+	}
+
+	// copy and sort to prevent mutation of original array
+	aggregationIndices = slices.quickSort(aggregationIndices)
+
+	for j:=len(aggregationIndices)-1; j>=0; j-- {
+		splice(groupIndices, aggregationIndices[j], 1)
+	}
+	return groupIndices
+}
+
 export ll.toStrict({
 	PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
 	PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
+	EMPTY_JSON_KEY: EMPTY_JSON_KEY,
 
 	xsvColumnId: xsvColumnId,
 	purifySpec: purifySpec,
-	decomposePfconvImportCfg: decomposePfconvImportCfg
+	decomposePfconvImportCfg: decomposePfconvImportCfg,
+	calculateGroupAxesIndices: calculateGroupAxesIndices
 })

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -83,7 +83,7 @@ getColumnSpec := func(axesSpec, col) {
  * @return map: reference - a reference to a map resource storing imported data.
  */
 importFile := func(xsvFile, xsvType, spec, ...ops) {
-	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
+	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types, found: " + xsvType)
 
 	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
 
@@ -294,7 +294,7 @@ exportFrame := func(pf, xsvType, params, ...ops) {
 		}
 		pf = pfm
 	}
-	
+
 	r := render.create(exportPfTpl, {
 		pf: pf,
 		xsvType: xsvType,

package/dist/tengo/lib/slices.lib.tengo

@@ -286,6 +286,20 @@ fromSet := func(set) {
 	return sortByFn(slice, func(elem) { return elem })
 }
 
+/**
+ * Normalizes a slice by ensuring it is never undefined.
+ * Returns an empty slice if input is undefined, otherwise returns the input slice unchanged.
+ *
+ * @param s: any[]|undefined - Input slice that may be undefined
+ * @returns any[] - Normalized slice, never undefined
+ */
+normalize := func(s) {
+	if is_undefined(s) {
+		return []
+	}
+	return s
+}
+
 export ll.toStrict({
 	length: length,
 	safeAppend: safeAppend,
@@ -301,5 +315,6 @@ export ll.toStrict({
 	quickSortInPlaceFn: quickSortInPlaceFn,
 	sortByFn: sortByFn,
 	fromSet: fromSet,
-	slice: slice
+	slice: slice,
+	normalize: normalize
 })

package/dist/tengo/lib/validation.lib.tengo

@@ -118,7 +118,7 @@ _validateTypes := func(js, tags, keyPath, ...stack) {
 		return success
 	}
 
-	if k == "bool" && is_bool(js) {
+	if (k == "bool" || k == "boolean") && is_bool(js) {
 		return success
 	}
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "2.12.1",
+  "version": "2.13.0",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",