@platforma-sdk/workflow-tengo 5.20.1 → 5.21.0

package/dist/tengo/lib/pframes.process-column-common.lib.tengo

@@ -0,0 +1,366 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+builder := import("@platforma-sdk/workflow-tengo:pframes.builder")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+buildOutputTrace := func(opts, output, inputSpec) {
+	hasOutputTrace := !is_undefined(output.traceSteps)
+	hasGlobalTrace := !is_undefined(opts.traceSteps)
+	hasInputTrace := !is_undefined(inputSpec.annotations) && !is_undefined(inputSpec.annotations[pSpec.A_TRACE])
+
+	outputTraceSteps := slices.normalize(opts.traceSteps) + slices.normalize(output.traceSteps)
+
+	trace := undefined
+	if len(outputTraceSteps) > 0 || hasInputTrace {
+		trace = pSpec.makeTrace(inputSpec, outputTraceSteps...)
+	}
+
+	outputOverrideTrace := output.overrideTrace
+	if is_undefined(outputOverrideTrace) {
+		outputOverrideTrace = hasOutputTrace || hasGlobalTrace
+	}
+
+	return { trace: trace, overrideTrace: outputOverrideTrace }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+createProcessColumnResult := func(renderResult, outputsMap, outputSpecs) {
+	_isColumnar := func(output) {
+		return output.type == "Xsv" || output.type == "TsvContent"
+	}
+	_requireOutput := func(name) {
+		output := outputsMap[name]
+		ll.assert(!is_undefined(output), "unknown output: %v", name)
+		return output
+	}
+
+
+	_requireColumnArg := func(output, column) {
+		if _isColumnar(output) {
+			ll.assert(len(column) == 1,
+				"column name must be provided for %s output", output.type)
+			return column[0]
+		}
+		ll.assert(len(column) == 0,
+			"column name should not be provided for non-Xsv/TsvContent output")
+		return undefined
+	}
+	_requireColumnSpec := func(name, columnName) {
+		spec := outputSpecs[name][columnName]
+		ll.assert(!is_undefined(spec), "unknown column: %v", columnName)
+		return spec
+	}
+
+	self := undefined
+	self = ll.toStrict({
+		renderResult: renderResult,
+
+
+
+
+		outputSpec: func(name, ...column) {
+			output := _requireOutput(name)
+			columnName := _requireColumnArg(output, column)
+			if _isColumnar(output) {
+				return _requireColumnSpec(name, columnName)
+			}
+			return outputSpecs[name]
+		},
+
+
+
+
+		outputData: func(name, ...column) {
+			output := _requireOutput(name)
+			columnName := _requireColumnArg(output, column)
+			if _isColumnar(output) {
+				return renderResult.output(name + "/" + columnName)
+			}
+			return renderResult.output(name)
+		},
+
+
+
+
+		output: func(name, ...column) {
+			output := _requireOutput(name)
+			columnName := _requireColumnArg(output, column)
+			if _isColumnar(output) {
+				return {
+					data: renderResult.output(name + "/" + columnName),
+					spec: _requireColumnSpec(name, columnName)
+				}
+			}
+			return {
+				data: renderResult.output(name),
+				spec: outputSpecs[name]
+			}
+		},
+
+
+
+
+		xsvOutputFrame: func(name) {
+			output := _requireOutput(name)
+			ll.assert(_isColumnar(output),
+				"xsvOutputFrame can only be used with Xsv or TsvContent outputs")
+			pf := builder.pFrameBuilder()
+			for columnName, spec in outputSpecs[name] {
+				pf.add(columnName, spec, renderResult.output(name + "/" + columnName))
+			}
+			return pf.build()
+		},
+
+
+
+
+
+		addAllOutputsToBuilder: func(bld, ...separator) {
+			sep := "."
+			if len(separator) > 0 {
+				sep = separator[0]
+			}
+			maps.forEach(outputsMap, func(name, output) {
+				if _isColumnar(output) {
+					for columnName, spec in outputSpecs[name] {
+						bld.add(
+							name + sep + columnName,
+							spec,
+							renderResult.output(name + "/" + columnName)
+						)
+					}
+				} else {
+					bld.add(name, outputSpecs[name], renderResult.output(name))
+				}
+			})
+			return bld
+		},
+
+
+
+
+		allOutputsFrame: func(...xsvNameSeparator) {
+			pf := builder.pFrameBuilder()
+			self.addAllOutputsToBuilder(pf, xsvNameSeparator...)
+			return pf.build()
+		},
+
+
+
+
+		listOutputs: func() {
+			return maps.getKeys(outputsMap)
+		},
+
+
+
+
+
+		isXsvOutput: func(name) {
+			return _isColumnar(_requireOutput(name))
+		},
+
+
+
+
+		listXsvColumns: func(name) {
+			output := _requireOutput(name)
+			ll.assert(_isColumnar(output), "output is not Xsv or TsvContent: %v", name)
+			return maps.getKeys(outputSpecs[name])
+		},
+
+
+
+
+
+		addXsvOutputToBuilder: func(bld, outputName, prefix) {
+			for columnName in self.listXsvColumns(outputName) {
+				bld.add(
+					prefix + columnName,
+					self.outputSpec(outputName, columnName),
+					self.outputData(outputName, columnName)
+				)
+			}
+			return bld
+		}
+	})
+
+	return self
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+deriveHeaderFromSpec := func(spec) {
+	if !is_undefined(spec.annotations) {
+		label := spec.annotations[pSpec.A_LABEL]
+		if is_string(label) && label != "" {
+			return label
+		}
+	}
+	if is_string(spec.name) && spec.name != "" {
+		return spec.name
+	}
+	ll.panic("cannot derive header from spec: no label annotation or name found")
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+resolveColumnSrc := func(entry) {
+	src := entry.src
+	filter := undefined
+
+	if is_map(src) && src.__isPrimaryRef == "v1" {
+		filter = src.filter
+		src = src.column
+	}
+
+	ll.assert(is_map(src) && !is_undefined(src.spec) && !is_undefined(src.data),
+		"column src must be { spec, data }, got: %v", src)
+
+	header := entry.header
+	if is_undefined(header) {
+		header = deriveHeaderFromSpec(src.spec)
+	}
+
+	result := { src: src, header: header }
+	if !is_undefined(filter) {
+		result.filter = filter
+	}
+	return result
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+normalizeColumnEntries := func(input) {
+	if is_undefined(input) {
+		return []
+	}
+
+
+
+
+	_isBareColumn := func(m) { return !is_undefined(m.spec) && !is_undefined(m.data) }
+	_isPrimaryRef := func(m) { return m.__isPrimaryRef == "v1" }
+	_normalizeEntry := func(entry) {
+		ll.assert(is_map(entry), "unrecognized column entry: %v", entry)
+		if !is_undefined(entry.src) {
+			return resolveColumnSrc(entry)
+		}
+		ll.assert(_isBareColumn(entry) || _isPrimaryRef(entry),
+			"unrecognized column entry: %v", entry)
+		return resolveColumnSrc({ src: entry })
+	}
+
+
+	if is_map(input) {
+		input = [input]
+	}
+	ll.assert(is_array(input), "column entries must be an array, got: %v", input)
+
+	return slices.map(input, _normalizeEntry)
+}
+
+export ll.toStrict({
+	buildOutputTrace: buildOutputTrace,
+	createProcessColumnResult: createProcessColumnResult,
+	deriveHeaderFromSpec: deriveHeaderFromSpec,
+	resolveColumnSrc: resolveColumnSrc,
+	normalizeColumnEntries: normalizeColumnEntries
+})

package/dist/tengo/lib/pframes.process-pcolumn-batch-helpers.lib.tengo

@@ -0,0 +1,242 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+text := import("text")
+json := import("json")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+pUtil := import("@platforma-sdk/workflow-tengo:pframes.util")
+
+
+
+
+
+
+
+
+
+
+
+
+resolveData := func(data) {
+	if smart.isField(data) {
+		return data.getValue()
+	}
+	return data
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+classifyPColumnData := func(res) {
+	if res.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED) {
+		return { type: pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED, isBinary: true }
+	} else if res.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED) {
+		return { type: pConstants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED, isBinary: false }
+	} else if res.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_PARQUET_PARTITIONED) {
+		return { type: pConstants.RTYPE_P_COLUMN_DATA_PARQUET_PARTITIONED, isBinary: false }
+	} else if res.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_JSON) {
+		return { type: pConstants.RTYPE_P_COLUMN_DATA_JSON, isBinary: false, isJson: true }
+	}
+	ll.panic("Unsupported data type for batch mode grouping: %v", res)
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+recordsInGroup := func(groupRes) {
+	if is_undefined(groupRes) {
+		return 0
+	}
+	if groupRes.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_JSON) {
+		return len(groupRes.getDataAsJson().data)
+	}
+	if groupRes.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED) {
+
+		return len(groupRes.inputs()) / 2
+	}
+
+	return len(groupRes.inputs())
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+forEachOutputField := func(outputDefs, fn) {
+	for output in outputDefs {
+		if output.type == "ResourceMap" {
+			fn(output, output.name)
+		} else if output.type == "Xsv" {
+			for column in output.settings.columns {
+				id := pUtil.xsvColumnId(column)
+				fn(output, output.name + "/" + id)
+			}
+		}
+	}
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+groupColumnData := func(data, isolationIndices, batchKeyIndices, batchKeyLength, hasIsolation) {
+	res := resolveData(data)
+	if !hasIsolation {
+		return { "[]": res }
+	}
+	typeInfo := classifyPColumnData(res)
+	groups := {}
+
+	if typeInfo.isJson {
+		meta := res.getDataAsJson()
+		for sKey, value in meta.data {
+			keyArray := json.decode(sKey)
+
+			isolationKey := []
+			for i in isolationIndices {
+				isolationKey = append(isolationKey, keyArray[i])
+			}
+			inGroupKey := []
+			for i in batchKeyIndices {
+				inGroupKey = append(inGroupKey, keyArray[i])
+			}
+
+			sIsolationKey := json.encode(isolationKey)
+			sInGroupKey := json.encode(inGroupKey)
+
+			group := groups[sIsolationKey]
+			if is_undefined(group) {
+				group = {}
+				groups[sIsolationKey] = group
+			}
+			group[sInGroupKey] = value
+		}
+		for sIsolationKey, groupData in groups {
+			groups[sIsolationKey] = smart.structBuilder(
+				pConstants.RTYPE_P_COLUMN_DATA_JSON,
+				json.encode({ keyLength: batchKeyLength, data: groupData })
+			).lockAndBuild()
+		}
+	} else {
+
+
+
+
+
+
+		resInputs := res.inputs()
+		for sKey in maps.getKeys(resInputs) {
+			field := resInputs[sKey]
+			sKeyP := sKey
+			suffix := ""
+			if typeInfo.isBinary {
+				dotIdx := text.last_index(sKey, ".")
+				ll.assert(dotIdx != -1, "BinaryPartitioned key must contain a dot: %v", sKey)
+				suffix = sKey[dotIdx:]
+				sKeyP = sKey[:dotIdx]
+			}
+			keyArray := json.decode(sKeyP)
+
+			isolationKey := []
+			for i in isolationIndices {
+				isolationKey = append(isolationKey, keyArray[i])
+			}
+			inGroupKey := []
+			for i in batchKeyIndices {
+				ll.assert(i < len(keyArray),
+					"malformed partition key %v: fewer components than the axes spec (index %v out of range)", sKeyP, i)
+				inGroupKey = append(inGroupKey, keyArray[i])
+			}
+
+			sIsolationKey := json.encode(isolationKey)
+			sInGroupKey := json.encode(inGroupKey)
+
+			group := groups[sIsolationKey]
+			if is_undefined(group) {
+				group = smart.structBuilder(
+					typeInfo.type,
+					json.encode({ partitionKeyLength: batchKeyLength })
+				)
+				groups[sIsolationKey] = group
+			}
+			group.createInputField(sInGroupKey + suffix).set(field)
+		}
+		for sIsolationKey in maps.getKeys(groups) {
+			groups[sIsolationKey] = groups[sIsolationKey].lockAndBuild()
+		}
+	}
+
+	return groups
+}
+
+export ll.toStrict({
+	resolveData: resolveData,
+	classifyPColumnData: classifyPColumnData,
+	recordsInGroup: recordsInGroup,
+	forEachOutputField: forEachOutputField,
+	groupColumnData: groupColumnData
+})