@platforma-sdk/workflow-tengo 1.2.6

package/dist/tengo/lib/maps.lib.tengo

@@ -0,0 +1,102 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+
+/**
+ * Returns a slice of keys of the map.
+ *
+ * @param map map[string]any - map to extract keys from
+ * @return result: string[] - a slice of keys.
+ */
+getKeys := func(map) {
+	res := []
+	for k, _ in map {
+		res = append(res, k)
+	}
+
+	return slices.sortByFn(res, func(elem) { return elem })
+}
+
+/**
+ * Returns true if map contains specified key. Note: current implementation is O(N)
+ */
+containsKey := func(map, key) {
+	for k, _ in map {
+		if key == k {
+			return true
+		}
+	}
+
+	return false
+}
+
+/**
+ * Transforms all keys of the map with a function and returns a new map with these keys.
+ *
+ * @param map: map[key]any
+ * @param fnK: func(key): newKey - a function that gets a key and returns a new key.
+ * @return new map: map[newKey]any - a transformed map
+ */
+mapKeys := func(map, fnK) {
+	newM := {}
+	for k, v in map {
+		newM[fnK(k)] = v
+	}
+
+	return newM
+}
+
+// Map values
+mapValues := func(map, fnV) {
+	newM := {}
+	for k, v in map {
+		newM[k] = fnV(v)
+	}
+
+	return newM
+}
+
+// Map keys and values
+map := func(map, fn) {
+	newM := {}
+	for k, v in map {
+		r := fn(k, v)
+		newM[r.key] = r.value
+	}
+
+	return newM
+}
+
+filter := func(map, predicateFn) {
+	newM := {}
+	for k, v in map {
+		if predicateFn(k, v) {
+			newM[k] = v
+		}
+	}
+
+	return newM
+}
+
+filterValues := func(map, predicateFnV) {
+	return filter(map, func(k, v) { return predicateFnV(v) })
+}
+
+filterKeys := func(map, predicateFnK) {
+	return filter(map, func(k, v) { return predicateFnK(k) })
+}
+
+filterKeysValues := func(map, predicateFnK, predicateFnV) {
+	return filter(map, func(k, v) { return predicateFnK(k) && predicateFnV(v) })
+}
+
+export ll.toStrict({
+	map : map,
+	mapKeys : mapKeys,
+	mapValues : mapValues,
+	filter : filter,
+	filterValues : filterValues,
+	filterKeys : filterKeys,
+	filterKeysValues : filterKeysValues,
+	getKeys: getKeys,
+	containsKey: containsKey
+})

package/dist/tengo/lib/objects.lib.tengo

@@ -0,0 +1,22 @@
+/**
+* Utilities for working with objects
+*/
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+/**
+* Deletes entries with undefined values from object
+*/
+deleteUndefined := func(obj) {
+    for k, v in obj {
+        if is_undefined(v) {
+            delete(obj, k)
+        }
+    }
+
+    return obj
+}
+
+export ll.toStrict({
+    deleteUndefined: deleteUndefined
+})

package/dist/tengo/lib/oop.lib.tengo

@@ -0,0 +1,29 @@
+/**
+ * Utility functions for working with objects.
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+/**
+ * Inherit all methods and properties of a super object
+ *
+ * @param super a parent from which we get all methods
+ * @param self an object in which we set all super methods
+ * @return self
+ */
+inherit := func(super, self) {
+	self["super"] = super
+
+	// inherit super methods
+	for k, v in super {
+		if self[k] == undefined {
+			self[k] = v
+		}
+	}
+
+	return ll.toStrict(self)
+}
+
+export ll.toStrict({
+	inherit: inherit
+})

package/dist/tengo/lib/path.lib.tengo

@@ -0,0 +1,165 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+text := import("text")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+
+/**
+ * Joins path elements into a single path.
+ *
+ * @param elems: ...string - any number of elements.
+ */
+join := func(...elems) {
+	validation.assertJsonSchema(elems, ["string"])
+	return ll.filepathJoin(elems...)
+}
+
+/**
+ * Like join, joins elements to a single path,
+ * but can be used safely in regexps.
+ *
+ * @param elems: ...string - any number of elements.
+ */
+regexpJoin := func(...elems) {
+	validation.assertJsonSchema(elems, ["string"])
+	return ll.filepathRegexpJoin(elems...)
+}
+
+/**
+ * Joins elements of the filepath and canonizes it, i.e.
+ * semantically resolves ../ and ./ and changes slash
+ * to a pathSeparator.
+ *
+ * @param elems: ...string - any number of elements.
+ */
+canonize := func(...elems) {
+	validation.assertJsonSchema(elems, ["string"])
+	return ll.filepathCanonize(elems...)
+}
+
+/**
+ * Returns a path separator. "/" on Unix-like systems, "\\" on Windows.
+ *
+ * @return separator: string
+ */
+separator := ll.pathSeparator
+
+/**
+ * Trims a beginning of the file path following by a path separator.
+ *
+ * @param path: string - a file path
+ * @param prefix: string - the beginning of the file path
+ * @return trimmed path: string - a path without the beginning and a path separator.
+ */
+trimPrefix := func(path, prefix) {
+	validation.assertJsonSchema([path, prefix], ["string"])
+	return text.trim_prefix(path, prefix+separator)
+}
+
+_sepOrDefault := func(...sep) {
+	validation.assertJsonSchema(sep, [["or", "string", "char", "null"]])
+
+	if slices.length(sep) == 0 {
+		return separator
+	}
+
+	return sep[0]
+}
+
+/**
+ * Splits a path to path components.
+ * If it was a file name, the last component will be this name.
+ *
+ * @param path: string - the path
+ * @param sep: string - an optional path separator.
+ * @return components: []string - all directories and a file name at the end.
+ */
+split := func(path, ...sep) {
+	sep = _sepOrDefault(sep...)
+
+	return text.split(path, sep)
+}
+
+/**
+ * Gets all directories with their ancestors from the root.
+ * For example, for dir1/dir2/dir3/abc.txt it gives:
+ * ["dir1", "dir1/dir2", "dir1/dir2/dir3", "dir1/dir2/dir3/abc.txt"]
+ *
+ * @param path: string - the path
+ * @param separator: string - an optional separator.
+ * @return components: []string - all directories and a file name at the end from their root.
+ */
+getBasenameDirs := func(path, ...sep) {
+	validation.assertJsonSchema(path, "string")
+	sep = _sepOrDefault(sep...)
+
+	dirs := split(path, sep)
+
+	dirTails := []
+	for i := 1; i <= len(dirs); i++ {
+		dirTails = append(dirTails, text.join(dirs[:i], sep))
+	}
+
+	return dirTails
+}
+
+/**
+ * Gets an extension to the last "." symbol,
+ * or, if the name contains ".gz" or any other archive suffix,
+ * then till the next from the last "." symbol.
+ *
+ * @param path: string - a path
+ * @return extension: string - the extension without the leading "."
+ */
+getExtension := func(path, ...sep) {
+	validation.assertJsonSchema(path, "string")
+	ext := ""
+	sep = _sepOrDefault(sep...)
+
+	nPoints := 1
+	if text.re_match(`\.(gz|xz|bzip|zstd)$`, path) {
+		nPoints = 2
+	}
+
+	for i := len(path) - 1; i >= 0 && path[i] != sep; i-- {
+		if path[i] == '.' {
+			ext = path[i:]
+			nPoints--
+			if nPoints == 0 {
+				break
+			}
+		}
+	}
+
+	if len(ext) > 0 {
+		ext = ext[1:] // remove leading "."
+	}
+
+	return ext
+}
+
+/**
+ * Checks if the path is correct.
+ *
+ * @param path: string - a path
+ */
+assertIsValid := func(path) {
+	validation.assertJsonSchema(path, "string")
+	if text.contains(path, "/") {
+		ll.assert(
+			text.contains(path, separator),
+			"path to a file must be separated by OS's specific path separator. " +
+				"Did you forget to use 'path.join' or 'path.regexpJoin'?")
+	}
+}
+
+export ll.toStrict({
+	join: join,
+	regexpJoin: regexpJoin,
+	canonize: canonize,
+	separator: separator,
+	trimPrefix: trimPrefix,
+	split: split,
+	getBasenameDirs: getBasenameDirs,
+	getExtension: getExtension,
+	assertIsValid: assertIsValid
+})

package/dist/tengo/lib/pframes.constants.lib.tengo

@@ -0,0 +1,53 @@
+/**
+ * Contain field names and other constant used in PFrames
+ * libraries and templates.
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+/**
+ * Name of the field under which the iteration value like in "map" and "aggregate" is passed
+ * to the iteration body template.
+ */
+VALUE_FIELD_NAME := "__value__"
+
+/**
+ * Name of the field under which the iteration key like in "map" and "aggregate" is passed
+ * to the iteration body template.
+ */
+KEY_FIELD_NAME := "__key__"
+
+RTYPE_P_FRAME :=  { Name: "PFrame", Version: "1" }
+
+P_COLUMN_DATA_TYPE_PREFIX := "PColumnData/"
+RTYPE_P_COLUMN_DATA_JSON := { Name: P_COLUMN_DATA_TYPE_PREFIX + "Json", Version: "1" }
+RTYPE_P_COLUMN_DATA_BINARY := { Name: P_COLUMN_DATA_TYPE_PREFIX + "Binary", Version: "1" }
+RTYPE_P_COLUMN_DATA_RESOURCE_MAP := { Name: P_COLUMN_DATA_TYPE_PREFIX + "ResourceMap", Version: "1" }
+RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED := { Name: P_COLUMN_DATA_TYPE_PREFIX + "Partitioned/ResourceMap", Version: "1" }
+RTYPE_P_COLUMN_DATA_JSON_PARTITIONED := { Name: P_COLUMN_DATA_TYPE_PREFIX + "JsonPartitioned", Version: "1" }
+RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED := { Name: P_COLUMN_DATA_TYPE_PREFIX + "BinaryPartitioned", Version: "1" }
+
+P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX := P_COLUMN_DATA_TYPE_PREFIX + "Partitioned/"
+RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED := { Name: P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX + "JsonPartitioned", Version: "1" }
+RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED := { Name: P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX + "BinaryPartitioned", Version: "1" }
+
+KIND_P_COLUMN := "PColumn"
+
+export ll.toStrict({
+	KIND_P_COLUMN                                 : KIND_P_COLUMN,
+	KEY_FIELD_NAME                                : KEY_FIELD_NAME,
+	VALUE_FIELD_NAME                              : VALUE_FIELD_NAME,
+	P_COLUMN_DATA_TYPE_PREFIX                     : P_COLUMN_DATA_TYPE_PREFIX,
+	P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX   : P_COLUMN_SUPER_PARTITIONED_DATA_TYPE_PREFIX,
+
+	RTYPE_P_FRAME                                 : RTYPE_P_FRAME,
+
+	RTYPE_P_COLUMN_DATA_JSON                      : RTYPE_P_COLUMN_DATA_JSON,
+	RTYPE_P_COLUMN_DATA_BINARY                    : RTYPE_P_COLUMN_DATA_BINARY,
+	RTYPE_P_COLUMN_DATA_RESOURCE_MAP              : RTYPE_P_COLUMN_DATA_RESOURCE_MAP,
+	RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED  : RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED,
+	RTYPE_P_COLUMN_DATA_JSON_PARTITIONED          : RTYPE_P_COLUMN_DATA_JSON_PARTITIONED,
+	RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED        : RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED,
+	RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED    : RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED,
+	RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED  : RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED
+})

package/dist/tengo/lib/pframes.export-pcolumn.lib.tengo

@@ -0,0 +1,47 @@
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+render := import("@platforma-sdk/workflow-tengo:render")
+file := import("@platforma-sdk/workflow-tengo:file")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+
+// reference to the template doing nested super-partitioned export
+nestedTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-single-pcolumn")
+
+/** Implements core export logic for export-single-pcolumn and export-multiple-pcolumns templates */
+exportPColumnData := func(data) {
+	if data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP_PARTITIONED) ||
+		data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED) ||
+		data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED) {
+		// super-partitioned data
+
+		newData := smart.structBuilder(data.info().Type, data.info().Data)
+		for key, partData in data.inputs() {
+			newData.createInputField(key).set(render.createEphemeral(nestedTpl, { data: partData }).output("result"))
+		}
+		return newData.lockAndBuild()
+
+	} else if data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP) ||
+		data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED) ||
+		data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED) {
+		// data directly referencing blob objects
+
+		newData := smart.structBuilder(data.info().Type, data.info().Data)
+		for key, partData in data.inputs() {
+			newData.createInputField(key).set(file.exportFile(partData))
+		}
+		return newData.lockAndBuild()
+
+	} else if data.checkResourceType(pConstants.RTYPE_P_COLUMN_DATA_JSON) {
+		// data embeds all information and needs no special exporting procedure
+
+		return data
+
+	} else {
+		ll.panic("unknonw resource type: %v", data.info().Type)
+	}
+}
+
+export ll.toStrict({
+	exportPColumnData: exportPColumnData
+})

package/dist/tengo/lib/pframes.lib.tengo

@@ -0,0 +1,84 @@
+/**
+ * Working with p-frames
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+json := import("json")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+render := import("@platforma-sdk/workflow-tengo:render")
+constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+
+/**
+ * Given a reference to the p-frame, converts all inner resource files into resources that can be
+ * downloaded from the UI side.
+ */
+exportFrame := func(pf) {
+    return render.createEphemeral(assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-pframe"), { pf: pf }).output("result")
+}
+
+/**
+ * Export single PColumn data
+ */
+exportColumnData := func(data) {
+    return render.createEphemeral(assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-single-pcolumn"), { data: data }).output("result")
+}
+
+/** 
+ * Creates a builder for p-frame resource.
+ */
+pFrameBuilder := func() {
+    r := smart.structBuilder(constants.RTYPE_P_FRAME)
+
+    self := undefined
+    self = ll.toStrict(oop.inherit(r, {
+
+        add: func(column, spec, data) {
+            if (!is_undefined(spec)) {
+                r.createInputField(column + ".spec").setRefOrJson(spec)
+            }
+            if (!is_undefined(data)) {
+		        r.createInputField(column + ".data").setRefOrJson(data)
+            }
+            return self
+        },
+
+        build: func() {
+            return r.lockAndBuild()
+        }
+    }))
+    return self
+}
+
+/** 
+ * Creates a builder for a resource map p-column's data.
+ */
+pColumnResourceMapDataBuilder := func(keyLength) {
+    r := smart.structBuilder(
+		constants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP,
+		json.encode({ keyLength: 1 })
+	)
+
+    self := undefined
+    self = ll.toStrict(oop.inherit(r, {
+
+        add: func(key, data) {
+            ll.assert(is_array(key), "key must be an array, got ", key)
+            r.createInputField(json.encode(key)).setRefOrJson(data)
+            return self
+        },
+
+        build: func() {
+            return r.lockAndBuild()
+        }
+    }))
+    return self
+}
+
+export ll.toStrict({
+    exportFrame: exportFrame,
+    pFrameBuilder: pFrameBuilder,
+    exportColumnData: exportColumnData,
+    pColumnResourceMapDataBuilder: pColumnResourceMapDataBuilder
+})

package/dist/tengo/lib/pframes.ll.lib.tengo

@@ -0,0 +1,86 @@
+/**
+ * Set of basic operations for PColumn data payload
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+render := import("@platforma-sdk/workflow-tengo:render")
+pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+
+llAggTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.aggregate")
+
+/**
+ * Instantiate an aggregation pipeline for the given inputs, and returns rendered ephemeral template.
+ * Use r.output("output_name") method to get corresponding result, where r is an object returned by
+ * this method.
+ *
+ * Contract for the body template:
+ *
+ *   Inputs:
+ *     "__key__"   - will be set to the json representation of the group key (i.e. keys in all
+ *                   positions not specified in the indices parameter)
+ *     "__value__" - "PColumnData/ResourceMap" resource with all the records in the current group
+ *     ...extra    - all values or references passed to optional last parameter of this method
+ *                   will be forwarded to the template as is
+ *
+ *  Outputs:
+ *    array of output names to collect from aggregation operations is specified as a parameter
+ *    of this operation
+ *
+ * @param data    data resource (currently only PColumnData/ResourceMap is supported)
+ * @param indices indices to aggregate over (these indices will be stripped from the input map,
+ *                meaning of this parameter is the opposite of GROUP BY from SQL)
+ * @param bodyTpl reference to the template implementing particular aggregation (see contract above)
+ * @param outputs array of output settings to to collect from rendered templates
+ *
+ *                  Common fields for all subtypes:
+ *                     {
+ *                       name:  string     - name of the output
+ *                       path?: string[]   - [optional] overrides the path inside the body template
+ *                                           result, first name is interpreted as output name. If
+ *                                           not specified path equals to `[name]`.
+ *                     }
+ *
+ *                  Variants:
+ *                    | { type: "Resource" } // - assume normal singular resource is produced by
+ *                                                this output
+ *                    | {
+ *                        type: "ResourceMap",
+ *                        keyLength: number // - expected key length of result from each operation
+ *                      } // - assume each record is ResourceMap
+ *                    | {
+ *                        type: "BinaryPartitioned" | "JsonPartitioned",
+ *                        partitionKeyLength: number // - expected partition length of result from
+ *                                                        each operation
+ *                      } // - assume each record is a partitionded Json or Binary PColumn data itself
+ *
+ * @param isEph   if true, body template will be rendered as ephemeral
+ * @param extra   optional map of parameters to pass to the template
+ */
+aggregate := func(data, indices, bodyTpl, outputs, isEph, ...extra) {
+	renderInputs := {
+		params: smart.createJsonResource({
+			indices: indices,
+			eph: isEph,
+			outputs: outputs
+		}),
+		body: bodyTpl,
+		data: data
+	}
+
+	if len(extra) == 1 {
+		for k, v in extra[0] {
+			renderInputs["__extra_" + k] = v
+		}
+	} else if len(extra) != 0 {
+		ll.panic("Wrong number of parameters for aggregate method.")
+	}
+
+	renderResult := render.createEphemeral(llAggTpl, renderInputs)
+	return renderResult;
+}
+
+export ll.toStrict({
+	aggregate: aggregate
+})