@platforma-sdk/workflow-tengo 5.7.2 → 5.8.0

package/dist/tengo/lib/annotations.lib.tengo

@@ -50,7 +50,7 @@ computeAnnotations := func(inputs) {
 	
 	columnsFromExpressions := util.extractColumnsFromExpressions(slices.map(steps, func(step) { return step.value }))
 	additionalColumns := is_array(inputs.additionalColumns) ? inputs.additionalColumns : []
-	columns := columnsFromExpressions + additionalColumns
+	columns := slices.unique(columnsFromExpressions + additionalColumns)
 	
 	bundleBuilder := wf.createPBundleBuilder()
 	bundleBuilder.addAnchor("main", anchorRef)

package/dist/tengo/lib/exec.constants.lib.tengo

@@ -11,6 +11,10 @@ MEDIUM_QUEUE   := "medium"
 LIGHT_QUEUE    := "light"
 UI_TASKS_QUEUE := "ui-tasks"
 
+
+RUNNER_BATCH    := "batch"    // batch system execution (k8s jobs, etc.)
+RUNNER_EXECUTOR := "executor" // local binary execution
+
 export ll.toStrict({
 	ARG_TYPE_STRING: ARG_TYPE_STRING,
 	ARG_TYPE_EXPRESSION: ARG_TYPE_EXPRESSION,
@@ -21,5 +25,8 @@ export ll.toStrict({
 	UI_TASKS_QUEUE: UI_TASKS_QUEUE,
 	LIGHT_QUEUE: LIGHT_QUEUE,
 	MEDIUM_QUEUE: MEDIUM_QUEUE,
-	HEAVY_QUEUE: HEAVY_QUEUE
+	HEAVY_QUEUE: HEAVY_QUEUE,
+
+	RUNNER_BATCH: RUNNER_BATCH,
+	RUNNER_EXECUTOR: RUNNER_EXECUTOR
 })

package/dist/tengo/lib/exec.limits.lib.tengo

@@ -23,8 +23,18 @@ _RTYPE_QUOTA := { Name: "Quota", Version: "1" }
 
 
 
-useBatch := func(queueName) {
-	return feats.hasBatch && !slices.hasElement([ constants.UI_TASKS_QUEUE ], queueName)
+getRunnerType := func(queueName) {
+
+	runner := feats.getRunnerFor(queueName)
+	if !is_undefined(runner) {
+		return runner
+	}
+
+
+	if feats.hasBatch && !slices.hasElement([ constants.UI_TASKS_QUEUE ], queueName) {
+		return constants.RUNNER_BATCH
+	}
+	return constants.RUNNER_EXECUTOR
 }
 
 
@@ -158,8 +168,9 @@ storageSpaceRequestBuilder := func() {
 	return self
 }
 
-_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1  := { Name: "ComputeRequest/executor", Version: "1" }
-_RTYPE_COMPUTE_REQUEST_BATCH_V1     := { Name: "ComputeRequest/batch", Version: "1" }
+_computeRequestResourceType := func(runnerType) {
+	return { Name: "ComputeRequest/" + runnerType, Version: "1" }
+}
 
 _CR_FIELD_REQUEST                   := "request"    // input
 _CR_FIELD_ALLOCATION                := "allocation" // output
@@ -205,13 +216,8 @@ computeRequestBuilder := func() {
 			ll.assert(smart.isReference(quota), "exec.limits.build: quota of ComputeRequest must be a valid reference")
 			ll.assert(queue_name != "", "exec.limits.build: queue name is required for computeRequestBuilder")
 
-			requestBuilder := undefined
-
-			if useBatch(queue_name) {
-				requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_BATCH_V1)
-			} else {
-				requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1)
-			}
+			runnerType := getRunnerType(queue_name)
+			requestBuilder := smart.ephemeralBuilder(_computeRequestResourceType(runnerType))
 
 			requestBuilder.getField(_CR_FIELD_REQUEST).set(quota)
 
@@ -229,7 +235,7 @@ computeRequestBuilder := func() {
 }
 
 export ll.toStrict({
-	useBatch:                        useBatch,
+	getRunnerType:                   getRunnerType,
 	quotaBuilder:                    quotaBuilder,
 	storageSpaceRequestBuilder:      storageSpaceRequestBuilder,
 	computeRequestBuilder:           computeRequestBuilder

package/dist/tengo/lib/exec.runcmd.lib.tengo

@@ -19,8 +19,10 @@ limits := import("@platforma-sdk/workflow-tengo:exec.limits")
 delayedCompAlloc := import("@platforma-sdk/workflow-tengo:exec.delayed-compute-allocation")
 
 _RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
-_RTYPE_RUN_COMMAND_EXEC_V2 := { Name: "RunCommand/executor", Version: "2" }
-_RTYPE_RUN_COMMAND_BATCH_V2 := { Name: "RunCommand/batch", Version: "2" }
+
+_runCommandResourceType := func(runnerType) {
+	return { Name: "RunCommand/" + runnerType, Version: "2" }
+}
 _RTYPE_RUN_COMMAND_CMD := { Name: "RunCommandCmd", Version: "1" }
 _RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
 _RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
@@ -669,17 +671,8 @@ builder := func(workdir, ...initialState) {
 
 			ll.assert(!is_undefined(queue), "runcmd.builder.build: queue is required for any command run")
 
-			useBatch := limits.useBatch(queue)
-
-			if useBatch {
-
-
-
-				runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_BATCH_V2)
-
-			} else {
-				runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_EXEC_V2)
-			}
+			runnerType := limits.getRunnerType(queue)
+			runRes = smart.ephemeralBuilder(_runCommandResourceType(runnerType))
 
 			runRes.getField(_FIELD_WORKDIR_IN).set(workdir)
 

package/dist/tengo/lib/feats.lib.tengo

@@ -30,10 +30,25 @@ _hasRunner := func(e_id) {
 	return !is_undefined(_executors[e_id])
 }
 
+
+_hasGetRunnerFor := ll.methodExists(plapi, "getRunnerFor")
+
+
+
+_getRunnerFor := func(queueName) {
+	if !_hasGetRunnerFor {
+		return undefined
+	}
+	return plapi.getRunnerFor(queueName)
+}
+
 export ll.toStrict({
 	hasExec:                    _hasRunner("executor"),     // local binary execution is available
 	hasBatch:                   _hasRunner("batch"),        // batch system execution is available
 
+
+	getRunnerFor:               _getRunnerFor,
+
 	pureFutureFields:           _isEnabled("pureFutureFields")   || _api >= 1, // 'pure' future fields magic that provides deduplication
 	commandExpressions:         _isEnabled("commandExpressions") || _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )
 

package/dist/tengo/lib/pframes.lib.tengo

@@ -1106,5 +1106,6 @@ export ll.toStrict({
 	pColumnResourceMapDataBuilder: pColumnResourceMapDataBuilder,
 	tsvFileBuilder: func() { return xsvBuilder.xsvFileBuilder("tsv") },
 	csvFileBuilder: func() { return xsvBuilder.xsvFileBuilder("csv") },
+	parquetFileBuilder: func() { return xsvBuilder.xsvFileBuilder("parquet") },
 	parseToJson: parseToJson
 })

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -798,6 +798,11 @@ isStructuredPfMap := func(pf) {
 	return true
 }
 
+validateXsvType := func(xsvType) {
+	validation.assertType(xsvType, "string,regex=csv|tsv|parquet",
+		"xsvType must be one of [csv, tsv, parquet], found: %v", xsvType)
+}
+
 export ll.toStrict({
 	PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
 	PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
@@ -810,5 +815,6 @@ export ll.toStrict({
 	pFrameToColumnsMap: pFrameToColumnsMap,
 	addColumnToWd: addColumnToWd,
 	isFlatPfMap: isFlatPfMap,
-	isStructuredPfMap: isStructuredPfMap
+	isStructuredPfMap: isStructuredPfMap,
+	validateXsvType: validateXsvType
 })

package/dist/tengo/lib/pframes.xsv-builder.lib.tengo

@@ -8,9 +8,10 @@ pBuilder := import("@platforma-sdk/workflow-tengo:pframes.builder")
 xsv := import("@platforma-sdk/workflow-tengo:pframes.xsv")
 canonical := import("@platforma-sdk/workflow-tengo:canonical")
 execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
+util := import("@platforma-sdk/workflow-tengo:pframes.util")
 
 xsvFileBuilder := func(xsvType) {
-    ll.assert(xsvType == "csv" || xsvType == "tsv", "xsvType must be 'csv' or 'tsv'")
+    util.validateXsvType(xsvType)
 
     PTablerNA := "%%NA%%" // Constant for PTabler NA representation
     naValueInternal := ""  // Default NA value, can be changed by setNA

package/dist/tengo/lib/pframes.xsv-import-file.lib.tengo

@@ -84,7 +84,7 @@ getColumnSpec := func(axesSpec, col) {
 
 
 importFile := func(xsvFile, xsvType, spec, ...ops) {
-	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types, found: " + xsvType)
+	util.validateXsvType(xsvType)
 
 	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
 
@@ -106,6 +106,8 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
 	ops := renderOps({ dataOnly: false, splitDataAndSpec: false, queue: execConstants.LIGHT_QUEUE, inputCache: undefined }, ops...)
 
 	if ops.transpose != undefined {
+		ll.assert(xsvType != "parquet",
+			"transpose is not supported with xsvType 'parquet', please use 'csv' or 'tsv' instead")
 
 		tops := ops.transpose
 
@@ -170,6 +172,8 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
 	if storageFormat == "Parquet" {
 		return xsvImportPt.importFileParquet(xsvFile, xsvType, spec, ops)
 	}
+	ll.assert(xsvType != "parquet",
+		"storageFormat '%v' is not supported with xsvType 'parquet', please use storageFormat 'Parquet'", storageFormat)
 
 
 	pureSpec := util.purifySpec(spec)

package/dist/tengo/lib/pframes.xsv-import-pt.lib.tengo

@@ -59,9 +59,14 @@ applyNaRegex := func(colExpr, naRegex, colType) {
 
 
 importFileParquet := func(xsvFile, xsvType, spec, ops) {
-	ll.assert(xsvType == "csv" || xsvType == "tsv", "unknown xsv type: %v", xsvType)
+	util.validateXsvType(xsvType)
 
 	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
+	if xsvType == "parquet" {
+		ll.assert(is_undefined(spec.separator), "separator is not supported with xsvType 'parquet'")
+		ll.assert(is_undefined(spec.commentLinePrefix), "commentLinePrefix is not supported with xsvType 'parquet'")
+		ll.assert(is_undefined(spec.skipEmptyLines), "skipEmptyLines is not supported with xsvType 'parquet'")
+	}
 
 
 	ll.assert(is_undefined(spec.index), "index axis is not supported with Parquet storage format")

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -89,7 +89,8 @@ getColumnSpec := func(axesSpec, col) {
 
 
 importFile := func(xsvFile, xsvType, spec, ...ops) {
-	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types, found: " + xsvType)
+	ll.assert(xsvType == "csv" || xsvType == "tsv" || xsvType == "parquet",
+		"expected one of [csv, tsv, parquet] types, found: %v", xsvType)
 
 	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
 

package/dist/tengo/lib/pt.lib.tengo

@@ -15,6 +15,7 @@ pframesSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
 render := import("@platforma-sdk/workflow-tengo:render")
 maps := import("@platforma-sdk/workflow-tengo:maps")
 canonical := import("@platforma-sdk/workflow-tengo:canonical")
+validation := import("@platforma-sdk/workflow-tengo:validation")
 
 
 
@@ -468,17 +469,14 @@ workflow := func() {
 
 
 			if !is_undefined(opts.format) {
-				if opts.format != "csv" && opts.format != "tsv" && opts.format != "ndjson" {
-					ll.panic("format must be 'csv', 'tsv', or 'ndjson'. Got: %v", opts.format)
-				}
+				validation.assertType(opts.format, "string,regex=csv|tsv|parquet|ndjson",
+					"format must be one of [csv, tsv, parquet, ndjson], found: %v", opts.format)
 				if !is_undefined(finalFormat) && finalFormat != opts.format {
 					ll.panic("format and xsvType from structural input cannot be used together. Got: %v and %v", opts.format, finalFormat)
 				}
 				finalFormat = opts.format
 			} else if !is_undefined(opts.xsvType) {
-				if opts.xsvType != "csv" && opts.xsvType != "tsv" {
-					ll.panic("xsvType must be 'csv' or 'tsv'. Got: %v", opts.xsvType)
-				}
+				pframesUtil.validateXsvType(opts.xsvType)
 				if !is_undefined(finalFormat) && finalFormat != opts.xsvType {
 					ll.panic("opts.xsvType and xsvType from structural input cannot be used together. Got: %v and %v", opts.xsvType, finalFormat)
 				}
@@ -594,23 +592,23 @@ workflow := func() {
 					translation: translation
 				}
 			} else if finalFormat == "parquet" {
-					step = {
-						type: "read_parquet",
-						file: finalFileName,
-						name: finalDataFrameId
-					}
-				} else if finalFormat == "ndjson" {
-					step = {
-						type: "read_ndjson",
-						file: finalFileName,
-						name: finalDataFrameId
-					}
-				} else { // csv or tsv
-					step = {
-						type: "read_csv",
-						file: finalFileName,
-						name: finalDataFrameId
-					}
+				step = {
+					type: "read_parquet",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
+			} else if finalFormat == "ndjson" {
+				step = {
+					type: "read_ndjson",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
+			} else { // csv or tsv
+				step = {
+					type: "read_csv",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
 
 				if finalFormat == "csv" {
 					step.delimiter = ","
@@ -632,7 +630,7 @@ workflow := func() {
 			}
 
 
-			if !is_undefined(finalSchemaToUse) {
+			if !is_undefined(finalSchemaToUse) && finalFormat != "parquet" {
 				step.schema = finalSchemaToUse
 			}
 
@@ -1044,14 +1042,11 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 		if !is_undefined(opts.format) {
-			if opts.format != "csv" && opts.format != "tsv" && opts.format != "ndjson" {
-				ll.panic("format must be 'csv', 'tsv', or 'ndjson'. Got: %v", opts.format)
-			}
+			validation.assertType(opts.format, "string,regex=csv|tsv|parquet|ndjson",
+				"format must be one of [csv, tsv, parquet, ndjson], found: %v", opts.format)
 			finalFormat = opts.format
 		} else if !is_undefined(opts.xsvType) {
-			if opts.xsvType != "csv" && opts.xsvType != "tsv" {
-				ll.panic("xsvType must be 'csv' or 'tsv'. Got: %v", opts.xsvType)
-			}
+			pframesUtil.validateXsvType(opts.xsvType)
 			finalFormat = opts.xsvType
 		} else {
 

package/dist/tengo/lib/slices.lib.tengo

@@ -376,6 +376,24 @@ sortUnique := func(list) {
     return quickSortInPlace(uniqueList)
 }
 
+
+
+
+
+
+
+unique := func(list) {
+	arr := []
+	map := {}
+	for _, item in list {
+		if is_undefined(map[item]) {
+			arr = append(arr, item)
+			map[item] = true
+		}
+	}
+	return arr
+}
+
 export ll.toStrict({
 	length: length,
 	safeAppend: safeAppend,
@@ -396,5 +414,6 @@ export ll.toStrict({
 	fromSet: fromSet,
 	slice: slice,
 	normalize: normalize,
-	sortUnique: sortUnique
+	sortUnique: sortUnique,
+	unique: unique
 })

package/dist/tengo/lib/workdir.lib.tengo

@@ -323,7 +323,7 @@ _fill := func() {
 					quotaBuilder.ram(ram)
 				}
 
-				if limits.useBatch(queue) {
+				if limits.getRunnerType(queue) == execConstants.RUNNER_BATCH {
 					quotaBuilder.queue(execConstants.BATCH_QUEUE)
 				} else {
 					quotaBuilder.queue(queue)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "5.7.2",
+  "version": "5.8.0",
   "description": "Tengo SDK library",
   "type": "module",
   "types": "./dist/index.d.ts",
@@ -17,12 +17,12 @@
   ],
   "dependencies": {
     "@milaboratories/software-pframes-conv": "2.2.9",
-    "@platforma-open/milaboratories.software-small-binaries": "^1.15.25",
-    "@platforma-open/milaboratories.software-ptabler": "1.14.0",
-    "@platforma-open/milaboratories.software-ptexter": "1.2.0"
+    "@platforma-open/milaboratories.software-small-binaries": "^1.15.26",
+    "@platforma-open/milaboratories.software-ptexter": "1.2.0",
+    "@platforma-open/milaboratories.software-ptabler": "1.14.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "2.4.2"
+    "@platforma-sdk/tengo-builder": "2.4.8"
   },
   "scripts": {
     "type-check": "pl-tengo check",