@platforma-sdk/workflow-tengo 4.15.0 → 4.16.0

package/dist/tengo/lib/pframes.data.lib.tengo

@@ -98,9 +98,12 @@ parseData := func(inputParam, ...optsRaw) {
         state.keyLength = resourceData.keyLength
         state.relevantTopLayerKeyLength = state.keyLength
     } else if state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED ||
-              state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED ||
-              state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED {
+              state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED {
+        state.partitionKeyLength = resourceData.partitionKeyLength
+        state.relevantTopLayerKeyLength = state.partitionKeyLength
+    } else if state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED {
         state.partitionKeyLength = resourceData.partitionKeyLength
+        state.keyLength = resourceData.keyLength
         state.relevantTopLayerKeyLength = state.partitionKeyLength
     } else if state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED ||
               state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED {
@@ -188,26 +191,86 @@ _createPColumnDataInstance = func(state) {
     self := undefined
 
     self = ll.toStrict({
+
+
+
+
+
         resource: state.resource,
 
+
+
+
+
         parsed: state.parsed,
 
+
+
+
+
+
+
+
         getSpec: func() {
             return state.spec
         },
 
+
+
+
+
+
         getResourceType: func() {
             return state.resourceType
         },
 
+
+
+
+
         keyLength: state.keyLength,
+
+
+
+
+
+
         partitionKeyLength: state.partitionKeyLength,
+
+
+
+
+
         superPartitionKeyLength: state.superPartitionKeyLength,
 
+
+
+
+
+
+
+
+
         data: state.data,
 
+
+
+
+
+
+
+
         parseKey: _createDecodeKeyString,
 
+
+
+
+
+
+
+
+
+
         uniqueKeyValues: func(axisIdx) {
             if !state.parsed {
                 ll.panic("uniqueKeyValues: resource is not parsed.")
@@ -224,6 +287,17 @@ _createPColumnDataInstance = func(state) {
             return maps.values(uniqueValsMap)
         },
 
+
+
+
+
+
+
+
+
+
+
+
         createDataResource: func() {
             if !is_undefined(state.resource) {
 
@@ -271,6 +345,27 @@ _createPColumnDataInstance = func(state) {
             return builder.lockAndBuild()
         },
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
         filter: func(slicingParams) {
             if len(slicingParams) == 0 {
                 return self
@@ -465,6 +560,26 @@ _createPColumnDataInstance = func(state) {
                 ll.panic("Filtering not supported for type %s", state.resourceType)
             }
         },
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
         partition: func(...axesToPartitionBy) {
             if !state.parsed {
@@ -511,6 +626,20 @@ _createPColumnDataInstance = func(state) {
             return resultPartitions
         },
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
         forEach: func(userCallback) {
             if !state.parsed {
                 ll.panic("forEach: resource is not parsed.")
@@ -525,6 +654,61 @@ _createPColumnDataInstance = func(state) {
             })
         },
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        forEachPartition: func(axesToPartitionBy, userCallback) {
+            if !state.parsed {
+                ll.panic("forEachPartition: resource is not parsed.")
+            }
+            if !is_array(axesToPartitionBy) {
+                ll.panic("forEachPartition: first argument must be an array of axis indices.")
+            }
+            if !is_callable(userCallback) {
+                ll.panic("forEachPartition: second argument must be a function.")
+            }
+
+            partitions := self.partition(axesToPartitionBy...)
+
+            maps.forEach(partitions, func(partitionKeyStr, partitionPColumnData) {
+                decodedPartitionKey := json.decode(partitionKeyStr)
+                userCallback(decodedPartitionKey, partitionPColumnData)
+            })
+        },
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
         flatten: func() {
             if !state.parsed {
                 ll.panic("flatten: resource is not parsed.")
@@ -532,7 +716,7 @@ _createPColumnDataInstance = func(state) {
 
             isJsonSuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED
             isBinarySuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED
-            isResourceMapSuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_SUPER_PARTITIONED
+            isResourceMapSuper := state.resourceType == pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED
 
             if !(isJsonSuper || isBinarySuper || isResourceMapSuper) {
                 return self // Not a type that can be flattened

package/dist/tengo/lib/pt.expression.lib.tengo

@@ -323,6 +323,41 @@ _newExpression = func(expression, currentAlias) {
 		isNotNull: func() { return unaryOp("is_not_na") }, // Polars: is_not_null(), TS: is_not_na
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		structField: func(...fieldNames) {
+			ll.assert(len(fieldNames) > 0, "structField expects at least one field name")
+			for fieldName in fieldNames {
+				ll.assert(is_string(fieldName), "All field names must be strings, got: %T", fieldName)
+			}
+
+
+			currentExpr := expression
+			for fieldName in fieldNames {
+				currentExpr = {
+					type: "struct_field",
+					struct: currentExpr,
+					fields: fieldName
+				}
+			}
+
+			return _newExpression(currentExpr, undefined)
+		},
+
+
 		
 		strToUpper: func() { return unaryStrOp("to_upper") },
 		

package/dist/tengo/lib/pt.lib.tengo

@@ -1195,9 +1195,92 @@ concat := func(dataframes, ...optionsRaw) {
 	return _newDataFrame(parentWorkflow, outputDfName)
 }
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+treeJoin := func(dataframes, opts) {
+	ll.assert(is_array(dataframes), "First argument to treeJoin must be an array of DataFrame objects.")
+	ll.assert(len(dataframes) > 0, "DataFrame array for treeJoin cannot be empty.")
+
+	if !is_map(opts) || is_undefined(opts.on) {
+		ll.panic("treeJoin requires options with 'on' field")
+	}
+	if !is_string(opts.on) {
+		ll.panic("'on' option must be a string")
+	}
+
+
+	if len(dataframes) == 1 {
+		return dataframes[0]
+	}
+
+
+	parentWorkflow := undefined
+	for i, df in dataframes {
+		if is_undefined(df) || is_undefined(df._getDfName) || is_undefined(df._getWorkflow) {
+			ll.panic("Argument at index %d in dataframes array is not a valid DataFrame object.", i)
+		}
+		currentWorkflow := df._getWorkflow()
+		if is_undefined(parentWorkflow) {
+			parentWorkflow = currentWorkflow
+		} else if currentWorkflow._wfId != parentWorkflow._wfId {
+			ll.panic("All DataFrames in a treeJoin operation must belong to the same workflow.")
+		}
+	}
+
+	pfs := dataframes
+
+
+	for len(pfs) > 1 {
+		nextPfs := []
+
+		for i := 1; i < len(pfs); i += 2 {
+			pf1 := pfs[i - 1]
+			pf2 := pfs[i]
+			joined := pf1.join(pf2, { on: opts.on, how: "full", coalesce: true })
+			nextPfs = append(nextPfs, joined)
+		}
+
+		if len(pfs) % 2 == 1 {
+			nextPfs = append(nextPfs, pfs[len(pfs) - 1])
+		}
+		pfs = nextPfs
+	}
+
+	ll.assert(len(pfs) == 1, "Expected exactly one data frame, got " + string(len(pfs)))
+	return pfs[0]
+}
+
 export ll.toStrict({
 	workflow: workflow,
 	concat: concat,
+	treeJoin: treeJoin,
 
 	col: exp.col,
 	lit: exp.lit,

package/dist/tengo/lib/slices.lib.tengo

@@ -153,6 +153,42 @@ reduce := func(slice, fn, ...init) {
 }
 
 
+
+
+
+
+
+
+
+
+flatten := func(slice, ...opts) {
+	if is_undefined(slice) {
+		return []
+	}
+
+	deep := false
+	if len(opts) > 0 && !is_undefined(opts[0]) && !is_undefined(opts[0].deep) {
+		deep = opts[0].deep
+	}
+
+	result := []
+	for _, elem in slice {
+		if is_array(elem) {
+			if deep {
+				flattened := flatten(elem, {deep: true})
+				result = append(result, flattened...)
+			} else {
+				result = append(result, elem...)
+			}
+		} else {
+			ll.panic("flatten: expected array element but got %s", type_name(elem))
+		}
+	}
+
+	return result
+}
+
+
 every := func(slice, fn) {
 	for _, elem in slice {
 		if !fn(elem) {
@@ -322,6 +358,7 @@ export ll.toStrict({
 	filter: filter,
 	map: map,
 	reduce: reduce,
+	flatten: flatten,
 	every: every,
 	quickSortInPlace: quickSortInPlace,
 	quickSort: quickSort,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "4.15.0",
+  "version": "4.16.0",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -18,10 +18,10 @@
   "dependencies": {
     "@milaboratories/software-pframes-conv": "2.2.2",
     "@platforma-open/milaboratories.software-small-binaries": "^1.15.21",
-    "@platforma-open/milaboratories.software-ptabler": "1.9.0"
+    "@platforma-open/milaboratories.software-ptabler": "1.11.0"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^2.1.13"
+    "@platforma-sdk/tengo-builder": "^2.1.14"
   },
   "scripts": {
     "type-check": "pl-tengo check",