@platforma-sdk/workflow-tengo 4.14.1 → 4.16.0

package/dist/tengo/lib/pframes.data.lib.tengo

@@ -98,9 +98,12 @@ parseData := func(inputParam, ...optsRaw) {
         state.keyLength = resourceData.keyLength
         state.relevantTopLayerKeyLength = state.keyLength
     } else if state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED ||
-              state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED ||
-              state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED {
+              state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED {
+        state.partitionKeyLength = resourceData.partitionKeyLength
+        state.relevantTopLayerKeyLength = state.partitionKeyLength
+    } else if state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED {
         state.partitionKeyLength = resourceData.partitionKeyLength
+        state.keyLength = resourceData.keyLength
         state.relevantTopLayerKeyLength = state.partitionKeyLength
     } else if state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED ||
               state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED {
@@ -188,26 +191,86 @@ _createPColumnDataInstance = func(state) {
     self := undefined
 
     self = ll.toStrict({
+
+
+
+
+
         resource: state.resource,
 
+
+
+
+
         parsed: state.parsed,
 
+
+
+
+
+
+
+
         getSpec: func() {
             return state.spec
         },
 
+
+
+
+
+
         getResourceType: func() {
             return state.resourceType
         },
 
+
+
+
+
         keyLength: state.keyLength,
+
+
+
+
+
+
         partitionKeyLength: state.partitionKeyLength,
+
+
+
+
+
         superPartitionKeyLength: state.superPartitionKeyLength,
 
+
+
+
+
+
+
+
+
         data: state.data,
 
+
+
+
+
+
+
+
         parseKey: _createDecodeKeyString,
 
+
+
+
+
+
+
+
+
+
         uniqueKeyValues: func(axisIdx) {
             if !state.parsed {
                 ll.panic("uniqueKeyValues: resource is not parsed.")
@@ -224,6 +287,17 @@ _createPColumnDataInstance = func(state) {
             return maps.values(uniqueValsMap)
         },
 
+
+
+
+
+
+
+
+
+
+
+
         createDataResource: func() {
             if !is_undefined(state.resource) {
 
@@ -271,6 +345,27 @@ _createPColumnDataInstance = func(state) {
             return builder.lockAndBuild()
         },
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
         filter: func(slicingParams) {
             if len(slicingParams) == 0 {
                 return self
@@ -465,6 +560,26 @@ _createPColumnDataInstance = func(state) {
                 ll.panic("Filtering not supported for type %s", state.resourceType)
             }
         },
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
         partition: func(...axesToPartitionBy) {
             if !state.parsed {
@@ -511,6 +626,20 @@ _createPColumnDataInstance = func(state) {
             return resultPartitions
         },
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
         forEach: func(userCallback) {
             if !state.parsed {
                 ll.panic("forEach: resource is not parsed.")
@@ -525,6 +654,61 @@ _createPColumnDataInstance = func(state) {
             })
         },
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        forEachPartition: func(axesToPartitionBy, userCallback) {
+            if !state.parsed {
+                ll.panic("forEachPartition: resource is not parsed.")
+            }
+            if !is_array(axesToPartitionBy) {
+                ll.panic("forEachPartition: first argument must be an array of axis indices.")
+            }
+            if !is_callable(userCallback) {
+                ll.panic("forEachPartition: second argument must be a function.")
+            }
+
+            partitions := self.partition(axesToPartitionBy...)
+
+            maps.forEach(partitions, func(partitionKeyStr, partitionPColumnData) {
+                decodedPartitionKey := json.decode(partitionKeyStr)
+                userCallback(decodedPartitionKey, partitionPColumnData)
+            })
+        },
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
         flatten: func() {
             if !state.parsed {
                 ll.panic("flatten: resource is not parsed.")
@@ -532,7 +716,7 @@ _createPColumnDataInstance = func(state) {
 
             isJsonSuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED
             isBinarySuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED
-            isResourceMapSuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_SUPER_PARTITIONED
+            isResourceMapSuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED
 
             if !(isJsonSuper || isBinarySuper || isResourceMapSuper) {
                 return self // Not a type that can be flattened

package/dist/tengo/lib/pt.expression.lib.tengo

@@ -323,6 +323,41 @@ _newExpression = func(expression, currentAlias) {
 		isNotNull: func() { return unaryOp("is_not_na") }, // Polars: is_not_null(), TS: is_not_na
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		structField: func(...fieldNames) {
+			ll.assert(len(fieldNames) > 0, "structField expects at least one field name")
+			for fieldName in fieldNames {
+				ll.assert(is_string(fieldName), "All field names must be strings, got: %T", fieldName)
+			}
+
+
+			currentExpr := expression
+			for fieldName in fieldNames {
+				currentExpr = {
+					type: "struct_field",
+					struct: currentExpr,
+					fields: fieldName
+				}
+			}
+
+			return _newExpression(currentExpr, undefined)
+		},
+
+
 		
 		strToUpper: func() { return unaryStrOp("to_upper") },
 		

package/dist/tengo/lib/pt.lib.tengo

@@ -52,6 +52,19 @@ workflowCounter := 0
 
 
 
+_detectFormatFromFileName := func(fileName) {
+	if text.has_suffix(fileName, ".csv") {
+		return "csv"
+	} else if text.has_suffix(fileName, ".tsv") {
+		return "tsv"
+	} else if text.has_suffix(fileName, ".ndjson") || text.has_suffix(fileName, ".jsonl") {
+		return "ndjson"
+	}
+	return undefined // No format detected
+}
+
+
+
 
 
 
@@ -118,6 +131,16 @@ workflow := func() {
 
 
 
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -145,9 +168,11 @@ workflow := func() {
 			inputXsvType := undefined
 			finalDataFrameId := undefined
 			finalFileName := undefined
-			finalXsvType := undefined
+			finalFormat := undefined
 			inferSchemaOpt := undefined
 			optionsSchema := undefined
+			nRowsOpt := undefined
+			ignoreErrorsOpt := undefined
 
 			isStructuralInput := is_map(frameInput) && !is_undefined(frameInput.file)
 
@@ -162,7 +187,7 @@ workflow := func() {
 					inputSchema = frameInput.schema
 				}
 
-				finalXsvType = inputXsvType
+				finalFormat = inputXsvType
 			} else {
 				if smart.isReference(frameInput) {
 					fileRef = frameInput
@@ -172,15 +197,25 @@ workflow := func() {
 				}
 			}
 
-			ll.assert(is_undefined(finalXsvType) || is_undefined(opts.xsvType), "xsvType option cannot be used with structural input as xsvType is already defined in the input structure")
 
-			if !is_undefined(opts.xsvType) {
-				finalXsvType = opts.xsvType
+			if !is_undefined(opts.format) {
+				if opts.format != "csv" && opts.format != "tsv" && opts.format != "ndjson" {
+					ll.panic("format must be 'csv', 'tsv', or 'ndjson'. Got: %v", opts.format)
+				}
+				if !is_undefined(finalFormat) && finalFormat != opts.format {
+					ll.panic("format and xsvType from structural input cannot be used together. Got: %v and %v", opts.format, finalFormat)
+				}
+				finalFormat = opts.format
+			} else if !is_undefined(opts.xsvType) {
+				if opts.xsvType != "csv" && opts.xsvType != "tsv" {
+					ll.panic("xsvType must be 'csv' or 'tsv'. Got: %v", opts.xsvType)
+				}
+				if !is_undefined(finalFormat) && finalFormat != opts.xsvType {
+					ll.panic("opts.xsvType and xsvType from structural input cannot be used together. Got: %v and %v", opts.xsvType, finalFormat)
+				}
+				finalFormat = opts.xsvType
 			}
 
-			ll.assert(!is_undefined(finalXsvType), "xsvType option is required for direct file input")
-			ll.assert(finalXsvType == "csv" || finalXsvType == "tsv", "xsvType must be 'csv' or 'tsv'")
-
 			if !is_undefined(opts.id) {
 				finalDataFrameId = opts.id
 			} else {
@@ -189,11 +224,21 @@ workflow := func() {
 
 			if !is_undefined(opts.fileName) {
 				finalFileName = opts.fileName
+				if is_undefined(finalFormat) {
+					detectedFormat := _detectFormatFromFileName(finalFileName)
+					if !is_undefined(detectedFormat) {
+						finalFormat = detectedFormat
+					}
+				}
 			} else {
-				finalFileName = self._newAnonymousFileId(finalXsvType)
+				finalFileName = self._newAnonymousFileId(finalFormat)
 			}
 
 
+			if is_undefined(finalFormat) {
+				ll.panic("Unable to determine file format. Please specify 'format' or 'xsvType' option, or use a fileName with a recognizable extension (.csv, .tsv, .ndjson, .jsonl)")
+			}
+
 			if !is_undefined(opts.inferSchema) {
 				if !is_bool(opts.inferSchema) {
 					ll.panic("'inferSchema' option must be a boolean. Got: %T", opts.inferSchema)
@@ -201,7 +246,6 @@ workflow := func() {
 				inferSchemaOpt = opts.inferSchema
 			}
 
-
 			if !is_undefined(opts.schema) {
 				if !is_array(opts.schema) {
 					ll.panic("'schema' option (from options map) must be an array. Got: %T", opts.schema)
@@ -209,6 +253,21 @@ workflow := func() {
 				optionsSchema = opts.schema
 			}
 
+			if !is_undefined(opts.nRows) {
+				if !is_int(opts.nRows) || opts.nRows < 0 {
+					ll.panic("'nRows' option must be a non-negative integer. Got: %v", opts.nRows)
+				}
+				nRowsOpt = opts.nRows
+			}
+
+			if !is_undefined(opts.ignoreErrors) {
+				if !is_bool(opts.ignoreErrors) {
+					ll.panic("'ignoreErrors' option must be a boolean. Got: %T", opts.ignoreErrors)
+				}
+				ignoreErrorsOpt = opts.ignoreErrors
+			}
+
+
 			if !is_undefined(fileRef) && !smart.isReference(fileRef) {
 				ll.panic("frameInput.file (for structural input) or frameInput (for direct reference) must be a valid resource reference. Got: %v", fileRef)
 			} else if !is_undefined(fileContent) && !is_string(fileContent) {
@@ -221,16 +280,26 @@ workflow := func() {
 				inFiles = append(inFiles, { content: fileContent, name: finalFileName })
 			}
 
-			readCsvStep := {
-				type: "read_csv",
-				file: finalFileName,
-				name: finalDataFrameId
-			}
 
-			if finalXsvType == "csv" {
-				readCsvStep.delimiter = ","
-			} else if finalXsvType == "tsv" {
-				readCsvStep.delimiter = "\t"
+			step := undefined
+			if finalFormat == "ndjson" {
+				step = {
+					type: "read_ndjson",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
+			} else { // csv or tsv
+				step = {
+					type: "read_csv",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
+
+				if finalFormat == "csv" {
+					step.delimiter = ","
+				} else if finalFormat == "tsv" {
+					step.delimiter = "\t"
+				}
 			}
 
 
@@ -241,17 +310,24 @@ workflow := func() {
 				finalSchemaToUse = inputSchema
 			}
 
+
 			if !is_undefined(finalSchemaToUse) {
-				readCsvStep.schema = finalSchemaToUse
+				step.schema = finalSchemaToUse
 			}
 
+			if !is_undefined(inferSchemaOpt) && inferSchemaOpt == false {
+				step.inferSchema = false
+			}
 
+			if !is_undefined(ignoreErrorsOpt) && ignoreErrorsOpt == true {
+				step.ignoreErrors = true
+			}
 
-			if !is_undefined(inferSchemaOpt) && inferSchemaOpt == false {
-				readCsvStep.inferSchema = false
+			if !is_undefined(nRowsOpt) {
+				step.nRows = nRowsOpt
 			}
 
-			self.addRawStep(readCsvStep)
+			self.addRawStep(step)
 			return _newDataFrame(self, finalDataFrameId)
 		},
 
@@ -465,32 +541,52 @@ _newDataFrame = func(parentWorkflow, dfName) {
 			opts = options[0]
 		}
 
-		delimiter := undefined
-		if text.has_suffix(outputFile, ".csv") {
-			delimiter = ","
-		} else if text.has_suffix(outputFile, ".tsv") {
-			delimiter = "\t"
-		}
+		finalFormat := undefined
 
-		if !is_undefined(opts.xsvType) {
-			if opts.xsvType == "csv" {
-				delimiter = ","
-			} else if opts.xsvType == "tsv" {
-				delimiter = "\t"
-			} else {
-				ll.panic("Unsupported xsvType: %v", opts.xsvType)
+
+		if !is_undefined(opts.format) {
+			if opts.format != "csv" && opts.format != "tsv" && opts.format != "ndjson" {
+				ll.panic("format must be 'csv', 'tsv', or 'ndjson'. Got: %v", opts.format)
+			}
+			finalFormat = opts.format
+		} else if !is_undefined(opts.xsvType) {
+			if opts.xsvType != "csv" && opts.xsvType != "tsv" {
+				ll.panic("xsvType must be 'csv' or 'tsv'. Got: %v", opts.xsvType)
+			}
+			finalFormat = opts.xsvType
+		} else {
+
+			detectedFormat := _detectFormatFromFileName(outputFile)
+			if !is_undefined(detectedFormat) {
+				finalFormat = detectedFormat
 			}
 		}
 
-		if is_undefined(delimiter) {
-			ll.panic("Can't infer xsvType from outputFile extension, and xsvType is not specified in options")
+		if is_undefined(finalFormat) {
+			ll.panic("Unable to determine output format from file extension. Please specify 'format' or 'xsvType' option, or use a file name with a recognizable extension (.csv, .tsv, .ndjson, .jsonl)")
 		}
 
-		step := {
-			type: "write_csv",
-			table: dfName,
-			file: outputFile,
-			delimiter: delimiter
+		step := undefined
+		if finalFormat == "ndjson" {
+			step = {
+				type: "write_ndjson",
+				table: dfName,
+				file: outputFile
+			}
+		} else { // csv or tsv
+			delimiter := undefined
+			if finalFormat == "csv" {
+				delimiter = ","
+			} else if finalFormat == "tsv" {
+				delimiter = "\t"
+			}
+
+			step = {
+				type: "write_csv",
+				table: dfName,
+				file: outputFile,
+				delimiter: delimiter
+			}
 		}
 
 		if !is_undefined(opts.columns) {
@@ -589,6 +685,19 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -616,6 +725,14 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 
+
+
+
+
+
+
+
+
 
 
 
@@ -1078,9 +1195,92 @@ concat := func(dataframes, ...optionsRaw) {
 	return _newDataFrame(parentWorkflow, outputDfName)
 }
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+treeJoin := func(dataframes, opts) {
+	ll.assert(is_array(dataframes), "First argument to treeJoin must be an array of DataFrame objects.")
+	ll.assert(len(dataframes) > 0, "DataFrame array for treeJoin cannot be empty.")
+
+	if !is_map(opts) || is_undefined(opts.on) {
+		ll.panic("treeJoin requires options with 'on' field")
+	}
+	if !is_string(opts.on) {
+		ll.panic("'on' option must be a string")
+	}
+
+
+	if len(dataframes) == 1 {
+		return dataframes[0]
+	}
+
+
+	parentWorkflow := undefined
+	for i, df in dataframes {
+		if is_undefined(df) || is_undefined(df._getDfName) || is_undefined(df._getWorkflow) {
+			ll.panic("Argument at index %d in dataframes array is not a valid DataFrame object.", i)
+		}
+		currentWorkflow := df._getWorkflow()
+		if is_undefined(parentWorkflow) {
+			parentWorkflow = currentWorkflow
+		} else if currentWorkflow._wfId != parentWorkflow._wfId {
+			ll.panic("All DataFrames in a treeJoin operation must belong to the same workflow.")
+		}
+	}
+
+	pfs := dataframes
+
+
+	for len(pfs) > 1 {
+		nextPfs := []
+
+		for i := 1; i < len(pfs); i += 2 {
+			pf1 := pfs[i - 1]
+			pf2 := pfs[i]
+			joined := pf1.join(pf2, { on: opts.on, how: "full", coalesce: true })
+			nextPfs = append(nextPfs, joined)
+		}
+
+		if len(pfs) % 2 == 1 {
+			nextPfs = append(nextPfs, pfs[len(pfs) - 1])
+		}
+		pfs = nextPfs
+	}
+
+	ll.assert(len(pfs) == 1, "Expected exactly one data frame, got " + string(len(pfs)))
+	return pfs[0]
+}
+
 export ll.toStrict({
 	workflow: workflow,
 	concat: concat,
+	treeJoin: treeJoin,
 
 	col: exp.col,
 	lit: exp.lit,

package/dist/tengo/lib/slices.lib.tengo

@@ -153,6 +153,42 @@ reduce := func(slice, fn, ...init) {
 }
 
 
+
+
+
+
+
+
+
+
+flatten := func(slice, ...opts) {
+	if is_undefined(slice) {
+		return []
+	}
+
+	deep := false
+	if len(opts) > 0 && !is_undefined(opts[0]) && !is_undefined(opts[0].deep) {
+		deep = opts[0].deep
+	}
+
+	result := []
+	for _, elem in slice {
+		if is_array(elem) {
+			if deep {
+				flattened := flatten(elem, {deep: true})
+				result = append(result, flattened...)
+			} else {
+				result = append(result, elem...)
+			}
+		} else {
+			ll.panic("flatten: expected array element but got %s", type_name(elem))
+		}
+	}
+
+	return result
+}
+
+
 every := func(slice, fn) {
 	for _, elem in slice {
 		if !fn(elem) {
@@ -322,6 +358,7 @@ export ll.toStrict({
 	filter: filter,
 	map: map,
 	reduce: reduce,
+	flatten: flatten,
 	every: every,
 	quickSortInPlace: quickSortInPlace,
 	quickSort: quickSort,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "4.14.1",
+  "version": "4.16.0",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -18,10 +18,10 @@
   "dependencies": {
     "@milaboratories/software-pframes-conv": "2.2.2",
     "@platforma-open/milaboratories.software-small-binaries": "^1.15.21",
-    "@platforma-open/milaboratories.software-ptabler": "1.8.0"
+    "@platforma-open/milaboratories.software-ptabler": "1.11.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.1.13"
+    "@platforma-sdk/tengo-builder": "^2.1.14"
   },
   "scripts": {
     "type-check": "pl-tengo check",