@platforma-sdk/workflow-tengo 5.3.5 → 5.3.7

package/dist/index.cjs

@@ -1,5 +1,6 @@
 module.exports = { Templates: {
   'll.get-field': { type: 'from-file', path: require.resolve('./tengo/tpl/ll.get-field.plj.gz') },
+  'pframes.export-parquet-pcolumn-chunk': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz') },
   'pframes.export-single-pcolumn': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
   'pframes.import-dir': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.import-dir.plj.gz') },
   'pframes.remove-nulls': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.remove-nulls.plj.gz') },
@@ -28,10 +29,11 @@ module.exports = { Templates: {
   'pframes.parse-to-json': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.parse-to-json.plj.gz') },
   'exec.load-software.impl': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.load-software.impl.plj.gz') },
   'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
-  'pframes.xsv-import-if-not-null': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
   'pt.workflow-run': { type: 'from-file', path: require.resolve('./tengo/tpl/pt.workflow-run.plj.gz') },
   'check_network.run_hello_world': { type: 'from-file', path: require.resolve('./tengo/tpl/check_network.run_hello_world.plj.gz') },
   'check_network.run_hello_world_py': { type: 'from-file', path: require.resolve('./tengo/tpl/check_network.run_hello_world_py.plj.gz') },
+  'pframes.xsv-export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
+  'pframes.xsv-import-if-not-null': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
   'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') },
   'pframes.process-pcolumn-data': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') },

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "ll.get-field" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "pt.import-dir" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "anonymize.deanonymize" | "anonymize.impl" | "check_network.workdir_type_name_extractor" | "exec.delayed-compute-allocation.impl" | "exec.monetization_to_json" | "exec.python.install-deps" | "exec.python.venv-bin-path" | "pframes.export-multiple-pcolumns" | "pframes.slice-data" | "check_network.create_workdir_from_storage" | "check_network.download_blob" | "check_network.upload_blob" | "check_network.upload_template" | "exec.R.create-renv" | "exec.exec-impl" | "exec.monetization_info" | "exec.python.create-venv" | "pframes.parse-to-json" | "exec.load-software.impl" | "exec.exec" | "pframes.xsv-import-if-not-null" | "pt.workflow-run" | "check_network.run_hello_world" | "check_network.run_hello_world_py" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
+declare type TplName = "ll.get-field" | "pframes.export-parquet-pcolumn-chunk" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "pt.import-dir" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "anonymize.deanonymize" | "anonymize.impl" | "check_network.workdir_type_name_extractor" | "exec.delayed-compute-allocation.impl" | "exec.monetization_to_json" | "exec.python.install-deps" | "exec.python.venv-bin-path" | "pframes.export-multiple-pcolumns" | "pframes.slice-data" | "check_network.create_workdir_from_storage" | "check_network.download_blob" | "check_network.upload_blob" | "check_network.upload_template" | "exec.R.create-renv" | "exec.exec-impl" | "exec.monetization_info" | "exec.python.create-venv" | "pframes.parse-to-json" | "exec.load-software.impl" | "exec.exec" | "pt.workflow-run" | "check_network.run_hello_world" | "check_network.run_hello_world_py" | "pframes.xsv-export-pframe" | "pframes.xsv-import-if-not-null" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,6 +1,7 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'll.get-field': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/ll.get-field.plj.gz') },
+  'pframes.export-parquet-pcolumn-chunk': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz') },
   'pframes.export-single-pcolumn': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
   'pframes.import-dir': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.import-dir.plj.gz') },
   'pframes.remove-nulls': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.remove-nulls.plj.gz') },
@@ -29,10 +30,11 @@ export const Templates = {
   'pframes.parse-to-json': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.parse-to-json.plj.gz') },
   'exec.load-software.impl': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.load-software.impl.plj.gz') },
   'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
-  'pframes.xsv-import-if-not-null': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
   'pt.workflow-run': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pt.workflow-run.plj.gz') },
   'check_network.run_hello_world': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/check_network.run_hello_world.plj.gz') },
   'check_network.run_hello_world_py': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/check_network.run_hello_world_py.plj.gz') },
+  'pframes.xsv-export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
+  'pframes.xsv-import-if-not-null': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
   'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') },
   'pframes.process-pcolumn-data': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') },

package/dist/tengo/lib/pframes.export-pcolumn.lib.tengo

@@ -9,6 +9,8 @@ json := import("json")
 
 nestedTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-single-pcolumn")
 
+parquetChunkTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-parquet-pcolumn-chunk")
+
 
 exportPColumnData := func(data) {
 	ll.assert(data.info().InputsLocked, "column data resource not locked for export: %v", data.info().Type)
@@ -41,29 +43,11 @@ exportPColumnData := func(data) {
 
 		newData := smart.structBuilder(data.info().Type, data.info().Data)
 		for key, partData in data.inputs() {
-			partDataResource := partData.getValue()
-			ll.assert(
-				partDataResource.checkResourceType(pConstants.RTYPE_PARQUET_CHUNK),
-				"expected resource %v for axis key %v, got %v",
-				pConstants.RTYPE_PARQUET_CHUNK,
-				key,
-				partDataResource.info().Type
-			)
-			ll.assert(partDataResource.info().InputsLocked, "resource %v inputs not locked", partDataResource.info().Type)
-
-
-
-
-			chunkMetadata := partDataResource.getDataAsJson()
-			chunkMetadata.isDownloadable = true
-
-			parquetChunkResource := smart.structBuilder(partDataResource.info().Type, json.encode(chunkMetadata))
-
-			partDataFields := partDataResource.metaInputs()
-			parquetChunkResource.createMetaField("blob").set(file.exportFile(partDataFields["blob"]))
-			parquetChunkResource.createMetaField("mapping").set(partDataFields["mapping"])
-
-			newData.createInputField(key).set(parquetChunkResource.lockAndBuild())
+			exportedDataPart := render.createEphemeral(parquetChunkTpl, {
+				key: key,
+				data: partData
+			}).output("result")
+			newData.createInputField(key).set(exportedDataPart)
 		}
 		return newData.lockAndBuild()
 

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -177,7 +177,7 @@ PFCONV_IMPORT_CFG_SCHEMA := {
 	`columnNamePrefix,?`: `string`,
 
 
-	`storageFormat,?` : `string,regex=Binary|Json`,
+	`storageFormat,?` : `string,regex=Binary|Json|Parquet`,
 
 
 	`partitionKeyLength,?` : `number`
@@ -468,7 +468,7 @@ addColumnToWd := func(name, spec, data, execBuilder, ops) {
 	vt := spec.valueType
 	ll.assert(vt == "Int" || vt == "Long" || vt == "Float" || vt == "Double" || vt == "String",
 		"Unsupported value type: ", vt, spec)
-	
+
     ll.assert(!is_undefined(data) && smart.isResource(data), "data must be a resource, got: %v", data)
 
 	ll.assert(!is_undefined(execBuilder), "execBuilder must be defined")

package/dist/tengo/lib/pframes.xsv-import-file.lib.tengo

@@ -7,6 +7,7 @@ util := import("@platforma-sdk/workflow-tengo:pframes.util")
 json := import("json")
 constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
+xsvImportPt := import("@platforma-sdk/workflow-tengo:pframes.xsv-import-pt")
 
 transposeXsvSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:table-converter")
 pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
@@ -95,6 +96,11 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
 		len(spec.axes)
 	)
 
+	storageFormat := spec.storageFormat
+	if is_undefined(storageFormat) {
+		storageFormat = "Binary"
+	}
+
 	xsvFileName := "file." + xsvType
 
 	ops := renderOps({ dataOnly: false, splitDataAndSpec: false, queue: execConstants.LIGHT_QUEUE, inputCache: undefined }, ops...)
@@ -161,6 +167,10 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
 		xsvFile = transposeXsv.getFile(transposedFileName)
 	}
 
+	if storageFormat == "Parquet" {
+		return xsvImportPt.importFileParquet(xsvFile, xsvType, spec, ops)
+	}
+
 
 	pureSpec := util.purifySpec(spec)
 

package/dist/tengo/lib/pframes.xsv-import-pt.lib.tengo

@@ -0,0 +1,234 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+objects := import("@platforma-sdk/workflow-tengo:objects")
+util := import("@platforma-sdk/workflow-tengo:pframes.util")
+pt := import("@platforma-sdk/workflow-tengo:pt")
+execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+
+getColumnSpec := func(axesSpec, col) {
+	spec := maps.deepMerge(col.spec, {
+		kind: "PColumn",
+		axesSpec: axesSpec
+	})
+	return objects.deleteUndefined(spec)
+}
+
+applyPreProcessSteps := func(colExpr, preProcessSteps) {
+	result := colExpr
+	if !is_undefined(preProcessSteps) {
+		for step in preProcessSteps {
+			if step.type == "regexpExtract" {
+				result = result.extractEcmaRegex(step.pattern)
+			} else if step.type == "regexpReplace" {
+				result = result.replaceEcmaRegex(step.pattern, step.replacement)
+			} else {
+				ll.panic("unknown preProcess step type: %v", step.type)
+			}
+		}
+	}
+	return result
+}
+
+applyNaRegex := func(colExpr, naRegex, colType) {
+	if is_undefined(naRegex) {
+		return colExpr.cast(colType)
+	}
+	return pt.when(colExpr.matchesEcmaRegex(naRegex)).
+		then(pt.lit(undefined).cast(colType)).
+		otherwise(colExpr.cast(colType))
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+importFileParquet := func(xsvFile, xsvType, spec, ops) {
+	ll.assert(xsvType == "csv" || xsvType == "tsv", "unknown xsv type: %v", xsvType)
+
+	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
+
+
+	ll.assert(is_undefined(spec.index), "index axis is not supported with Parquet storage format")
+
+	ll.assert(is_undefined(spec.allowArtificialColumns), "allowArtificialColumns are not supported with Parquet storage format")
+
+	if !is_undefined(spec.separator) {
+		expectedSeparator := ","
+		if xsvType == "tsv" {
+			expectedSeparator = "\t"
+		}
+		ll.assert(spec.separator == expectedSeparator,
+			"separator does not match xsvType %v, expected '%v', got '%v'",
+			xsvType, expectedSeparator, spec.separator)
+	}
+
+	wf := pt.workflow()
+	if !is_undefined(ops.queue) {
+		if ops.queue == execConstants.HEAVY_QUEUE {
+			wf.inHeavyQueue()
+		} else if ops.queue == execConstants.MEDIUM_QUEUE {
+			wf.inMediumQueue()
+		} else if ops.queue == execConstants.LIGHT_QUEUE {
+			wf.inLightQueue()
+		} else if ops.queue == execConstants.UI_TASKS_QUEUE {
+			wf.inUiQueue()
+		} else {
+			ll.panic("Unsupported queue: %v", ops.queue)
+		}
+	}
+	if !is_undefined(ops.cpu) {
+		wf.cpu(ops.cpu)
+	}
+	if !is_undefined(ops.mem) {
+		wf.mem(ops.mem)
+	}
+	if !is_undefined(ops.inputCache) {
+		wf.cacheInputs(ops.inputCache)
+	}
+
+	schema := []
+	for ax in spec.axes {
+		schema = append(schema, {
+			column: ax.column,
+			type: "String"
+		})
+	}
+	for col in spec.columns {
+		schema = append(schema, {
+			column: col.column,
+			type: "String"
+		})
+	}
+	df := wf.frame(xsvFile, {
+		xsvType: xsvType,
+		ignoreErrors: spec.skipEmptyLines == true,
+		commentPrefix: spec.commentLinePrefix,
+		schema: schema
+	})
+
+	allColumns := []
+	for ax in spec.axes {
+		allColumns = append(allColumns, ax.column)
+	}
+	for col in spec.columns {
+		allColumns = append(allColumns, col.column)
+	}
+	df = df.select(allColumns...)
+
+	filterConditions := []
+	for ax in spec.axes {
+		if !is_undefined(ax.filterOutRegex) {
+			filterConditions = append(filterConditions, pt.col(ax.column).matchesEcmaRegex(ax.filterOutRegex).not())
+		}
+	}
+	for col in spec.columns {
+		if !is_undefined(col.filterOutRegex) {
+			filterConditions = append(filterConditions, pt.col(col.column).matchesEcmaRegex(col.filterOutRegex).not())
+		}
+	}
+	if len(filterConditions) > 0 {
+		df = df.filter(pt.and(filterConditions...))
+	}
+
+	projection := []
+	for ax in spec.axes {
+		colExpr := pt.col(ax.column)
+		colExpr = applyPreProcessSteps(colExpr, ax.preProcess)
+		colExpr = applyNaRegex(colExpr, ax.naRegex, ax.spec.type)
+		projection = append(projection, colExpr.alias(ax.column))
+	}
+	for col in spec.columns {
+		colExpr := pt.col(col.column)
+		colExpr = applyPreProcessSteps(colExpr, col.preProcess)
+		colExpr = applyNaRegex(colExpr, col.naRegex, col.spec.valueType)
+		projection = append(projection, colExpr.alias(col.column))
+	}
+	df = df.withColumns(projection...)
+
+	notNullConditions := []
+
+
+	for col in spec.columns {
+		if col.allowNA == false {
+
+			notNullConditions = append(notNullConditions, pt.col(col.column).isNotNull())
+		}
+	}
+	if len(notNullConditions) > 0 {
+		df = df.filter(pt.and(notNullConditions...))
+	}
+
+	saveParams := {
+		axes: [],
+		columns: [],
+		partitionKeyLength: 0
+	}
+	if !is_undefined(spec.partitionKeyLength) {
+		saveParams.partitionKeyLength = spec.partitionKeyLength
+	}
+	for ax in spec.axes {
+		saveParams.axes = append(saveParams.axes, {
+			column: ax.column,
+			spec: maps.deepMerge({ name: ax.column }, ax.spec)
+		})
+	}
+	for col in spec.columns {
+		saveParams.columns = append(saveParams.columns, {
+			column: col.column,
+			spec: maps.deepMerge({ name: col.column }, col.spec)
+		})
+	}
+
+	frameName := "xsv_import_output"
+	df.saveFrameDirect(frameName, saveParams)
+	pf := wf.run().getFrameDirect(frameName)
+
+	if ops.dataOnly {
+		pfd := smart.mapBuilder()
+		for col in spec.columns {
+			id := util.xsvColumnId(col)
+			field := pf.getFutureInputField(col.column + ".data")
+			pfd.createInputField(id).set(field)
+		}
+		return pfd.lockAndBuild()
+	}
+
+	result := {}
+	axesSpec := slices.map(spec.axes, func(ax) { return ax.spec })
+	for col in spec.columns {
+		colSpec := getColumnSpec(axesSpec, col)
+		data := pf.getFutureInputField(col.column + ".data")
+
+		id := util.xsvColumnId(col)
+		if ops.splitDataAndSpec {
+			result[id] = {
+				spec: colSpec,
+				data: data
+			}
+		} else {
+			result[id + ".spec"] = colSpec
+			result[id + ".data"] = data
+		}
+	}
+	return result
+}
+
+export ll.toStrict({
+	importFileParquet: importFileParquet
+})

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -10,15 +10,10 @@ render := import("@platforma-sdk/workflow-tengo:render")
 maps := import("@platforma-sdk/workflow-tengo:maps")
 util := import("@platforma-sdk/workflow-tengo:pframes.util")
 pBuilder := import("@platforma-sdk/workflow-tengo:pframes.builder")
-pframesSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 xsvIF := import("@platforma-sdk/workflow-tengo:pframes.xsv-import-file")
 text := import("text")
-execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
-pt := import("@platforma-sdk/workflow-tengo:pt")
-slices := import("@platforma-sdk/workflow-tengo:slices")
-canonical := import("@platforma-sdk/workflow-tengo:canonical")
 
 importXsvIfNullTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-if-not-null")
 
@@ -304,84 +299,16 @@ exportFrame := func(pf, xsvType, params, ...ops) {
 		ll.panic("Unexpected p-frame format: %s", pf)
 	}
 
-	resultFile := "result." + xsvType
-
-	columnEntries := []
-	rawColumns := []
-	maps.forEach(util.pFrameToColumnsMap(pf), func(frameUniqueColumnId, columnInfo) {
-		columnEntries = append(columnEntries, pt.p.column(frameUniqueColumnId, columnInfo))
-		rawColumns = append(rawColumns, { id: frameUniqueColumnId, spec: columnInfo.spec })
+	exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
+	r := render.create(exportPfTpl, {
+		pf: pf,
+		xsvType: xsvType,
+		params: partialParams
+	}, {
+		metaInputs: objects.deleteUndefined(runMetaInputs)
 	})
 
-	frameInput := undefined
-	if params.joinType == "Inner" {
-		frameInput = pt.p.inner(columnEntries...)
-	} else {
-		frameInput = pt.p.full(columnEntries...)
-	}
-
-	numericSelector := pt.sc.numeric()
-	if is_string(params.naStr) {
-		numericSelector = numericSelector.fillNull(params.naStr)
-	}
-	stringSelector := pt.sc.string()
-	if is_string(params.nullStr) {
-		stringSelector = stringSelector.fillNull(params.nullStr)
-	}
-
-	specDistiller := pframesSpec.createSpecDistiller(slices.map(rawColumns, func(c) { return c.spec }))
-	canonicalMap := {}
-	uniqueAxes := []
-	for column in rawColumns {
-		for axis in column.spec.axesSpec {
-			distilledAxis := specDistiller.distill(axis)
-			canonicalizedAxisSpec := canonical.encode(distilledAxis)
-			if is_undefined(canonicalMap[canonicalizedAxisSpec]) {
-				canonicalMap[canonicalizedAxisSpec] = true
-				distilledAxis.label = axis.annotations["pl7.app/label"]
-				uniqueAxes = append(uniqueAxes, distilledAxis)
-			}
-		}
-	}
-
-	columnSelectors := []
-	for axis in uniqueAxes {
-		label := axis.label
-		delete(axis, "label")
-		columnSelectors = append(columnSelectors, pt.sc.axis(axis).alias(label))
-	}
-	for column in rawColumns {
-		columnSelectors = append(columnSelectors, pt.col(column.id).alias(column.spec.annotations["pl7.app/label"]))
-	}
-
-	wf := pt.workflow()
-	if runMetaInputs.queue == execConstants.LIGHT_QUEUE {
-		wf = wf.inLightQueue()
-	} else if runMetaInputs.queue == execConstants.MEDIUM_QUEUE {
-		wf = wf.inMediumQueue()
-	} else if runMetaInputs.queue == execConstants.HEAVY_QUEUE {
-		wf = wf.inHeavyQueue()
-	} else if runMetaInputs.queue == execConstants.UI_TASKS_QUEUE {
-		wf = wf.inUiQueue()
-	} else if !is_undefined(runMetaInputs.queue) {
-		ll.panic("Unsupported queue: %v", runMetaInputs.queue)
-	}
-	if !is_undefined(runMetaInputs.cpu) {
-		wf = wf.cpu(runMetaInputs.cpu)
-	}
-	if !is_undefined(runMetaInputs.mem) {
-		wf = wf.mem(runMetaInputs.mem)
-	}
-	if !is_undefined(runMetaInputs.inputCache) {
-		wf = wf.cacheInputs(runMetaInputs.inputCache)
-	}
-	wf.frame(frameInput).
-		select(numericSelector, stringSelector).
-		select(columnSelectors...).
-		save(resultFile)
-
-	result := wf.run()
-	return result.getFile(resultFile)
+	return r.output("result")
 }
 
 export ll.toStrict({

package/dist/tengo/lib/pt.expression.lib.tengo

@@ -110,9 +110,11 @@ _newWindowExpression = func(expression, aggregation, currentAlias) {
 		_getExpression: func(ctx) {
 			ll.assert(is_map(ctx), "ctx must be a map")
 			ll.assert(!is_undefined(expression), "Using this operation as a window expression is not allowed. Use it as a group by expression instead. %v", aggregation)
-			ll.assert(!is_undefined(currentAlias), "Alias is not defined for this window expression.")
 
 			valueExpr := expression(ctx)
+			if is_undefined(currentAlias) {
+				return valueExpr
+			}
 			return {type: "alias", value: valueExpr, name: currentAlias}
 		},
 		
@@ -979,7 +981,11 @@ _newExpression = func(expression) {
 
 
 		matchesEcmaRegex: func(ecmaRegex) {
-			ll.assert(is_string(ecmaRegex) && len(ecmaRegex) > 0, "ecma_regex must be a non-empty string.")
+			ll.assert(is_string(ecmaRegex), "ecma_regex must be a string.")
+
+			if len(ecmaRegex) == 0 {
+				return self.eq("")
+			}
 
 			return _newExpression(func(ctx) {
 				valueExpr := self._getExpression(ctx)
@@ -1030,6 +1036,83 @@ _newExpression = func(expression) {
 					substitutionsOnly: substitutionsOnly
 				})
 			})
+		},
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		replaceEcmaRegex: func(ecmaRegex, replacement) {
+			ll.assert(is_string(ecmaRegex) && len(ecmaRegex) > 0, "ecmaRegex must be a non-empty string.")
+			ll.assert(is_string(replacement), "replacement must be a string.")
+
+			return _newExpression(func(ctx) {
+				valueExpr := self._getExpression(ctx)
+				return {
+					type: "replace_ecma_regex",
+					value: valueExpr,
+					ecmaRegex: ecmaRegex,
+					replacement: replacement
+				}
+			})
+		},
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		extractEcmaRegex: func(ecmaRegex) {
+			ll.assert(is_string(ecmaRegex) && len(ecmaRegex) > 0, "ecmaRegex must be a non-empty string.")
+
+			return _newExpression(func(ctx) {
+				valueExpr := self._getExpression(ctx)
+				return {
+					type: "extract_ecma_regex",
+					value: valueExpr,
+					ecmaRegex: ecmaRegex
+				}
+			})
 		}
 	})
 	return self

package/dist/tengo/lib/pt.lib.tengo

@@ -83,6 +83,8 @@ _isPEntry := func(entry) {
 	return ll.isMap(entry) && ll.methodExists(entry, "_getColumnsMap") && ll.methodExists(entry, "_getJoinTree");
 }
 
+_frameToPEntries := undefined
+
 slicedColumnCounter := 0
 
 p := ll.toStrict({
@@ -111,7 +113,7 @@ p := ll.toStrict({
 	slicedColumn: func(name, columnMap, axisFilters) {
 		_validateColumn(name, columnMap)
 		util.validateAxisFilters(axisFilters)
-		
+
 		slicedColumnCounter += 1
 		oldName := "p_sliced_" + slicedColumnCounter
 
@@ -131,31 +133,35 @@ p := ll.toStrict({
 
 	inner: func(...entries) {
 		ll.assert(len(entries) > 0, "inner join requires at least one entry")
-		
+
+		if len(entries) == 1 && !_isPEntry(entries[0]) {
+			entries = _frameToPEntries(entries[0])
+		}
+
 		mergedColMap := {}
 		joinTreeEntries := []
-		
+
 		for entry in entries {
 			ll.assert(_isPEntry(entry), "all entries must be PEntry objects")
-			
+
 			entryColMap := entry._getColumnsMap()
 			entryJoinTree := entry._getJoinTree()
-			
+
 			for colName, colInfo in entryColMap {
 				if !is_undefined(mergedColMap[colName]) {
 					ll.panic("column name %s is already used, please pick a different name", colName)
 				}
 				mergedColMap[colName] = colInfo
 			}
-			
+
 			joinTreeEntries = append(joinTreeEntries, entryJoinTree)
 		}
-		
+
 		joinTree := {
 			type: "inner",
 			entries: joinTreeEntries
 		}
-		
+
 		return _newPEntry(mergedColMap, joinTree)
 	},
 
@@ -165,31 +171,35 @@ p := ll.toStrict({
 
 	full: func(...entries) {
 		ll.assert(len(entries) > 0, "full join requires at least one entry")
-		
+
+		if len(entries) == 1 && !_isPEntry(entries[0]) {
+			entries = _frameToPEntries(entries[0])
+		}
+
 		mergedColMap := {}
 		joinTreeEntries := []
-		
+
 		for entry in entries {
 			ll.assert(_isPEntry(entry), "all entries must be PEntry objects")
-			
+
 			entryColMap := entry._getColumnsMap()
 			entryJoinTree := entry._getJoinTree()
-			
+
 			for colName, colInfo in entryColMap {
 				if !is_undefined(mergedColMap[colName]) {
 					ll.panic("column name %s is already used, please pick a different name", colName)
 				}
 				mergedColMap[colName] = colInfo
 			}
-			
+
 			joinTreeEntries = append(joinTreeEntries, entryJoinTree)
 		}
-		
+
 		joinTree := {
 			type: "full",
 			entries: joinTreeEntries
 		}
-		
+
 		return _newPEntry(mergedColMap, joinTree)
 	},
 
@@ -201,44 +211,75 @@ p := ll.toStrict({
 	outer: func(primary, ...secondary) {
 		ll.assert(_isPEntry(primary), "primary entry must be a PEntry object")
 		ll.assert(len(secondary) > 0, "outer join requires at least one secondary entry")
-		
+
 		mergedColMap := {}
-		
+
 		primaryColMap := primary._getColumnsMap()
 		primaryJoinTree := primary._getJoinTree()
-		
+
 		for colName, colInfo in primaryColMap {
 			mergedColMap[colName] = colInfo
 		}
-		
+
 		secondaryJoinTrees := []
-		
+
 		for entry in secondary {
 			ll.assert(_isPEntry(entry), "all secondary entries must be PEntry objects")
-			
+
 			entryColMap := entry._getColumnsMap()
 			entryJoinTree := entry._getJoinTree()
-			
+
 			for colName, colInfo in entryColMap {
 				if !is_undefined(mergedColMap[colName]) {
 					ll.panic("column name %s is already used, please pick a different name", colName)
 				}
 				mergedColMap[colName] = colInfo
 			}
-			
+
 			secondaryJoinTrees = append(secondaryJoinTrees, entryJoinTree)
 		}
-		
+
 		joinTree := {
 			type: "outer",
 			primary: primaryJoinTree,
 			secondary: secondaryJoinTrees
 		}
-		
+
 		return _newPEntry(mergedColMap, joinTree)
 	}
 })
 
+_frameToPEntries = func(frameInput) {
+	frameResource := undefined
+	if pframesUtil.isFlatPfMap(frameInput) {
+		frameResource = frameInput
+	} else if pframesUtil.isStructuredPfMap(frameInput) {
+		pfBuilder := pframesBuilder.pFrameBuilder()
+		for key, value in frameInput {
+			pfBuilder.add(key, value.spec, value.data)
+		}
+		frameResource = pfBuilder.build()
+	} else if smart.isReference(frameInput) {
+		if smart.isField(frameInput) {
+			ll.assert(frameInput.isSet(), "frameInput must be set")
+			frameResource = frameInput.getValue()
+		} else {
+			frameResource = frameInput
+		}
+		ll.assert(frameResource.checkResourceType(pframesConstants.RTYPE_P_FRAME), "frameResource must be a PFrame resource")
+	} else {
+		ll.panic("unsupported frameInput type: %v", frameInput)
+	}
+
+	columnsMap := pframesUtil.pFrameToColumnsMap(frameResource)
+	columnEntries := []
+	maps.forEach(columnsMap, func(frameUniqueColumnId, columnInfo) {
+		columnEntry := p.column(frameUniqueColumnId, columnInfo)
+		columnEntries = append(columnEntries, columnEntry)
+	})
+	return columnEntries
+}
+
 
 _detectFormatFromFileName := func(fileName) {
 	if text.has_suffix(fileName, ".csv") {
@@ -360,11 +401,12 @@ workflow := func() {
 
 
 
+
 
 
 		frame: func(frameInput, ...optionsRaw) {
 			ll.assert(!is_undefined(frameInput), "frameInput must be provided")
-			
+
 			opts := {}
 			if len(optionsRaw) > 0 {
 				if len(optionsRaw) == 1 && is_map(optionsRaw[0]) {
@@ -374,7 +416,6 @@ workflow := func() {
 				}
 			}
 
-			frameResource := undefined
 			fileRef := undefined
 			fileContent := undefined
 			inputSchema := undefined
@@ -386,6 +427,7 @@ workflow := func() {
 			optionsSchema := undefined
 			nRowsOpt := undefined
 			ignoreErrorsOpt := undefined
+			commentPrefixOpt := undefined
 
 			if is_map(frameInput) && !is_undefined(frameInput.file) {
 				fileRef = frameInput.file
@@ -400,28 +442,9 @@ workflow := func() {
 
 				finalFormat = inputXsvType
 			} else if smart.isReference(frameInput) {
-				frameInputResource := frameInput
-				if smart.isField(frameInputResource) {
-					frameInputResource = frameInputResource.getValue()
-				}
-				if frameInputResource.checkResourceType(pframesConstants.RTYPE_P_FRAME) {
-					frameResource = frameInputResource
-					finalFormat = "pframe"
-				} else {
-					fileRef = frameInput
-				}
+				fileRef = frameInput
 			} else if is_string(frameInput) || is_bytes(frameInput) {
 				fileContent = frameInput
-			} else if pframesUtil.isFlatPfMap(frameInput) {
-				frameResource = frameInput
-				finalFormat = "pframe"
-			} else if pframesUtil.isStructuredPfMap(frameInput) {
-				pfBuilder := pframesBuilder.pFrameBuilder()
-				for key, value in frameInput {
-					pfBuilder.add(key, value.spec, value.data)
-				}
-				frameResource = pfBuilder.build()
-				finalFormat = "pframe"
 			} else if _isPEntry(frameInput) {
 				finalFormat = "pentry"
 			} else {
@@ -498,6 +521,13 @@ workflow := func() {
 				ignoreErrorsOpt = opts.ignoreErrors
 			}
 
+			if !is_undefined(opts.commentPrefix) {
+				if !is_string(opts.commentPrefix) {
+					ll.panic("'commentPrefix' option must be a string. Got: %T", opts.commentPrefix)
+				}
+				commentPrefixOpt = opts.commentPrefix
+			}
+
 			if !is_undefined(fileRef) {
 				ll.assert(
 					smart.isReference(fileRef),
@@ -511,17 +541,7 @@ workflow := func() {
 
 
 		step := undefined
-		if finalFormat == "pentry" || finalFormat == "pframe" {
-			if finalFormat == "pframe" {
-				columnsMap := pframesUtil.pFrameToColumnsMap(frameResource)
-				columnEntries := []
-				for frameUniqueColumnId, columnInfo in columnsMap {
-					columnEntry := p.column(frameUniqueColumnId, columnInfo)
-					columnEntries = append(columnEntries, columnEntry)
-				}
-				frameInput = p.full(columnEntries...)
-			}
-
+		if finalFormat == "pentry" {
 			columnsMap := frameInput._getColumnsMap()
 			src := frameInput._getJoinTree()
 
@@ -577,13 +597,17 @@ workflow := func() {
 					name: finalDataFrameId
 				}
 
-				if finalFormat == "csv" {
-					step.delimiter = ","
-				} else if finalFormat == "tsv" {
-					step.delimiter = "\t"
-				}
+			if finalFormat == "csv" {
+				step.delimiter = ","
+			} else if finalFormat == "tsv" {
+				step.delimiter = "\t"
 			}
 
+			if !is_undefined(commentPrefixOpt) {
+				step.commentPrefix = commentPrefixOpt
+			}
+		}
+
 
 			finalSchemaToUse := undefined
 			if !is_undefined(optionsSchema) {
@@ -705,16 +729,17 @@ workflow := func() {
 			for _, frame in inFrames {
 				for _, columnInfo in frame.columns {
 					for axis in columnInfo.spec.axesSpec {
-						if is_undefined(canonicalMap[axis.name]) {
-							canonicalMap[axis.name] = {}
+						distilledAxis := specDistiller.distill(axis)
+						if is_undefined(canonicalMap[distilledAxis.name]) {
+							canonicalMap[distilledAxis.name] = {}
 						}
-						canonicalizedAxisSpec := canonical.encode(axis)
-						if is_undefined(canonicalMap[axis.name][canonicalizedAxisSpec]) {
-							canonicalMap[axis.name][canonicalizedAxisSpec] = true
-							if is_undefined(distilledAxesSpecs[axis.name]) {
-								distilledAxesSpecs[axis.name] = []
+						canonicalizedAxisSpec := canonical.encode(distilledAxis)
+						if is_undefined(canonicalMap[distilledAxis.name][canonicalizedAxisSpec]) {
+							canonicalMap[distilledAxis.name][canonicalizedAxisSpec] = true
+							if is_undefined(distilledAxesSpecs[distilledAxis.name]) {
+								distilledAxesSpecs[distilledAxis.name] = []
 							}
-							distilledAxesSpecs[axis.name] = append(distilledAxesSpecs[axis.name], axis)
+							distilledAxesSpecs[distilledAxis.name] = append(distilledAxesSpecs[distilledAxis.name], distilledAxis)
 						}
 					}
 				}
@@ -830,7 +855,7 @@ workflow := func() {
 					ll.panic("inFile must have either file or content, got: %v", inFile)
 				}
 			}
-			
+
 			outFrames := {}
 			for frameName, specsById in outFramesSpecs {
 				columnIds := []
@@ -947,7 +972,7 @@ _newDataFrameGroupBy = func(parentWorkflow, dfName, groupByExpressions) {
 					}
 					return aggExpr._getAggregation(ctx)
 				})
-				
+
 				return {
 					type: "aggregate",
 					inputTable: dfName,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "5.3.5",
+  "version": "5.3.7",
   "description": "Tengo SDK library",
   "type": "module",
   "types": "./dist/index.d.ts",
@@ -18,7 +18,7 @@
   "dependencies": {
     "@milaboratories/software-pframes-conv": "2.2.9",
     "@platforma-open/milaboratories.software-small-binaries": "^1.15.23",
-    "@platforma-open/milaboratories.software-ptabler": "1.12.6",
+    "@platforma-open/milaboratories.software-ptabler": "1.12.8",
     "@platforma-open/milaboratories.software-ptexter": "1.1.1"
   },
   "devDependencies": {