@platforma-sdk/workflow-tengo 2.13.6 → 2.15.0

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -1,24 +1,22 @@
-/**
- * Library providing xsv <> p-frames conversion utils.
- */
+
+
+
 
 validation := import("@platforma-sdk/workflow-tengo:validation")
 constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 objects := import("@platforma-sdk/workflow-tengo:objects")
 render := import("@platforma-sdk/workflow-tengo:render")
-exec := import("@platforma-sdk/workflow-tengo:exec")
-json := import("json")
-text := import("text")
+maps := import("@platforma-sdk/workflow-tengo:maps")
 util := import("@platforma-sdk/workflow-tengo:pframes.util")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
+xsvIF := import("@platforma-sdk/workflow-tengo:pframes.xsv-import-file")
+
+importXsvIfNullTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-if-not-null")
+
+
 
-transposeXsvSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:table-converter")
-pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
 
-/**
- * Calculate axes spec from pfconv spec
- */
 getAxesSpec := func(spec) {
 	axes := []
 	for ax in spec.axes {
@@ -55,168 +53,87 @@ getColumnSpec := func(axesSpec, col) {
 	return objects.deleteUndefined(spec)
 }
 
-/**
- * Imports xsv data into p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
- * provided by the spec used as map keys). Resulting p-columns will be always single-partitioned at most.
- *
- * @param xsvFile: reference - a reference to a file
- * @param xsvType: string - either csv or tsv
- * @param spec: object - xsv conversion specification
- * @param ops: object - additional options
- *               {
- *                 dataOnly: boolean  - set to true to completely skip creation of specs
- *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
- *                                              with "spec" + "data" fields in nested maps
- *                 transpose: object - if specified, then the input table will be transposed
- *                            {
- *                              separator: string - (optional) use separator instead of inferring from the file extension
- *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
- *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
- *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
- *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
- *                              valueName: string - (optional) the name of the value column; default "Value"
- *                              separatorOverride: string - (optional) specify separator in the output (optional)
- *                            }
- *               }
- * @return map: reference - a reference to a map resource storing imported data.
- */
-importFile := func(xsvFile, xsvType, spec, ...ops) {
-	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types, found: " + xsvType)
 
-	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
 
-	xsvFileName := "file." + xsvType
 
-	ops := renderOps({ dataOnly: false, splitDataAndSpec: false }, ops...)
 
-	if ops.transpose != undefined {
 
-		tops := ops.transpose
 
-		transposedFileName := "fileTransposed." + xsvType
 
-		// convert csv to p-frame and read resulting data
-		transposeXsv := exec.builder().
-			software(transposeXsvSw).
-			printErrStreamToStdout().
-			inLightQueue()
 
-		if (tops.separator != undefined) {
-			transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
-		}
-		if (tops.pAxisIdx != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
-		}
-		if (tops.pAxisName != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
-		}
-		if (tops.pAxisSearch != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
-		}
-		if (tops.pAxisNameOverride != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
-		}
-		if (tops.sAxisSearch != undefined) {
-			transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
-		}
-		if (tops.sAxisName != undefined) {
-			transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)
-		}
-		if (tops.valueName != undefined) {
-			transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)
-		}
-		if (tops.separatorOverride != undefined) {
-			transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)
-		}
 
-		transposeXsv = transposeXsv.
-			arg(xsvFileName).
-			arg(transposedFileName).
-			addFile(xsvFileName, xsvFile).
-			saveFile(transposedFileName).
-			run()
 
-		xsvFile = transposeXsv.getFile(transposedFileName)
-	}
 
-	// import p-frame from directory tpl
-	importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
-
-	// spec without any metadata (for caching purposes)
-	pureSpec := util.purifySpec(spec)
-
-	// convert csv to p-frame and read resulting data
-	pfconv := exec.builder().
-		software(pfconvSw).
-		printErrStreamToStdout().
-		inLightQueue().
-		arg("importCsv").
-		arg(xsvFileName).
-		arg("-p").arg("spec.json").
-		arg("-o").arg("out").
-		addFile(xsvFileName, xsvFile).
-		writeFile("spec.json", json.encode(pureSpec)).
-		processWorkdir("pf", importDirTpl, pureSpec).
-		run()
-
-	// p-columns data
-	pf := pfconv.getProcessorResult("pf")
-	if ops.dataOnly {
-		// @TODO discuss, as it changes the type of the return from map to object
-		return pf
-	}
 
-	axesSpec := getAxesSpec(spec)
 
-	result := {}
-	for col in spec.columns {
-		id := util.xsvColumnId(col)
 
-		if ops.splitDataAndSpec {
-			result[id] = {
-				data: pf.getFutureInputField(id),
-				spec: getColumnSpec(axesSpec, col)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+importFile := func(xsvFile, xsvType, spec, ...ops) {
+	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types, found: " + xsvType)
+
+	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
+
+	allowNullInput := false
+	if len(ops) > 0 {
+		allowNullInput = ops[0].allowNullInput
+	}
+
+	if allowNullInput {
+		return render.create(importXsvIfNullTpl, {
+			xsvFile: xsvFile,
+			params: {
+				xsvType: xsvType,
+				spec: spec,
+				ops: [maps.merge(ops[0], { allowNullInput: undefined })]
 			}
-		} else {
-			result[id + ".data"] = pf.getFutureInputField(id)
-			result[id + ".spec"] = getColumnSpec(axesSpec, col)
-		}
+		}).output("result")
 	}
 
-	return result
+	return xsvIF.importFile(xsvFile, xsvType, spec, ops...)
 }
 
-/**
- * Imports a map with xsv files into a p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
- * provided by the spec used as map keys). Resulting p-columns may be double-partitioned.
- *
- * @param xsvMap: reference - a reference to a p-column resource map with xsv files
- * @param xsvType: string - either csv or tsv
- * @param spec: object - xsv conversion specification
- * @param ops: object - additional options
- *               {
- *                 dataOnly: boolean  - set to true to completely skip creation of specs
- *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
- *                                              with "spec" + "data" fields in nested maps
- *                 additionalAxesSpec: AxisSpec[]  - array of additional axes spec to prepend to
- *                                                   each column spec
- *                 transpose: object - if specified, then the input table will be transposed
- *                            {
- *                              separator: string - (optional) use separator instead of inferring from the file extension
- *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
- *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
- *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
- *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
- *                              valueName: string - (optional) the name of the value column; default "Value"
- *                              separatorOverride: string - (optional) specify separator in the output (optional)
- *                            }
- *               }
- * @return map: reference - a reference to a map resource storing imported data.
- */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 
 	importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-map")
@@ -228,7 +145,7 @@ importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 		transpose: undefined
 	}, ops...)
 
-	// spec without any metadata (for caching purposes)
+
 	decomposedSpec := util.decomposePfconvImportCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
 	pureSpec := decomposedSpec.purifiedCfg
 	columnsSpec := decomposedSpec.columnsSpec
@@ -264,24 +181,24 @@ importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 	return result
 }
 
-/**
- * Export p-frame into xsv file
- *
- * @param pf: p-frame - a list of p-columns ({data, spec}) or a frame-like map of col.data col.spec entries
- * @param xsvType: string - "csv" or "tsv"
- * @param params: map - export params
- * @param ops: object - additional options
- *               {
- *                 partitions: {axisIdx: [keys]} - map of partitions to export; if not specified, all data is exported
- *				 }
- */
+
+
+
+
+
+
+
+
+
+
+
 exportFrame := func(pf, xsvType, params, ...ops) {
 	exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
 
 	ops := renderOps({}, ops...)
 
 	if is_array(pf) {
-		// transform into .data .spec map
+
 
 		pfm := {}
 		i := 0

package/dist/tengo/lib/render.lib.tengo

@@ -1,6 +1,6 @@
-/*
- * Create renderer objects to render templates from the code
- */
+
+
+
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
@@ -10,26 +10,27 @@ constants := import("@platforma-sdk/workflow-tengo:constants")
 oop := import("@platforma-sdk/workflow-tengo:oop")
 tx := import("tx")
 times := import("times")
+maps := import("@platforma-sdk/workflow-tengo:maps")
 
 
 _SUBSCRIPTION_FIELD := "subscription"
 
-/**
- * Creates a render template resource.
- *
- * @param tpl: template resource (see assets.importTemplate())
- * @param isEph: whether template is ephemeral
- * @param inputs: a map of inputs to be passed to the template
- * @param serviceInputs: (optional) a map of service inputs to be passed to the template.
- *                                  Service inputs are not accounted in dedupliation and recovering.
- *                                  Two resources with the same <inputs> and different <serviceInputs>
- *                                  would be considered as copies of each-other.
- * @return renderer: a smart resource with additional output("name") method
- */
+
+
+
+
+
+
+
+
+
+
+
+
 createUniversal := func(tpl, isEph, inputs, ...opts) {
 	serviceInputs := opts[0]
 
-	// build renderer resource
+
 	rbuilder := undefined
 	if isEph {
 		rbuilder = smart.ephemeralBuilder(constants.RTYPE_RENDER_EPH_TEMPLATE)
@@ -38,14 +39,14 @@ createUniversal := func(tpl, isEph, inputs, ...opts) {
 	}
 
 	if isEph {
-		// create internal subscription field
+
 		rbuilder.createField(_SUBSCRIPTION_FIELD, "OTW")
 	}
 
-	// template input
+
 	rbuilder.createInputField(tplutil.TEMPLATE_FIELD).set(tpl)
 
-	// inputs input
+
 	inputsMap := isEph ? smart.ephemeralMapBuilder() : smart.mapBuilder()
 	for name, value in inputs {
 		if !is_undefined(value) {
@@ -59,15 +60,15 @@ createUniversal := func(tpl, isEph, inputs, ...opts) {
 	}
 
 	rbuilder.createInputField(tplutil.INPUTS_MAP_FIELD).set(inputsMap.build())
-	// build
+
 	renderer := rbuilder.lockInputsAndBuild()
 
 	self := undefined
 	self = ll.toStrict(oop.inherit(renderer, {
 
-		/**
-		 * Returns an output field with a given name
-		 */
+
+
+
 		output: func(name, ...cache) {
 			ll.assert(len(cache) == 0 || is_int(cache[0]), "cache should be a number (use 'times' tengo library), got: %v", cache)
 			out := renderer.getFutureOutputField(tplutil.TEMPLATE_OUTPUT_PREFIX + name, isEph)
@@ -77,38 +78,92 @@ createUniversal := func(tpl, isEph, inputs, ...opts) {
 			}
 
 			return out
+		},
+
+
+
+
+		optionalOutput: func(name, ...cache) {
+			ll.assert(len(cache) == 0 || is_int(cache[0]), "cache should be a number (use 'times' tengo library), got: %v", cache)
+			out := renderer.getFutureFieldOptional(tplutil.TEMPLATE_OUTPUT_PREFIX + name, "output", isEph)
+			if len(cache) > 0 && cache[0] > 0 {
+				cache = cache[0]
+				tx.cacheSetToField(out.id, cache)
+			}
+
+			return out
+		},
+
+
+
+
+
+
+
+		resolveOutput: func(path) {
+			ll.assert(is_array(path), "path must be an array, got: %v", path)
+
+
+			pathCopy := []
+
+
+			if len(path) > 0 {
+				firstElem := path[0]
+
+
+				if is_string(firstElem) {
+					firstElem = { name: firstElem }
+				}
+
+
+				firstElemCopy := maps.clone(firstElem)
+				firstElemCopy.type = "output"
+
+
+				firstElemCopy.name = tplutil.TEMPLATE_OUTPUT_PREFIX + firstElemCopy.name
+
+				pathCopy = append(pathCopy, firstElemCopy)
+			}
+
+
+			for i := 1; i < len(path); i++ {
+				pathCopy = append(pathCopy, path[i])
+			}
+
+
+			return renderer.buildFutureField(pathCopy)
 		}
 	}))
 
 	return self
 }
 
-/**
- * Creates a render template resource.
- *
- * @param tpl: template resource (see assets.importTemplate())
- * @param inputs: a map of inputs to be passed to the template
- * @param serviceInputs: (optional) a map of service inputs to be passed to the template.
- *                                  Service inputs are not accounted in dedupliation and recovering.
- *                                  Two resources with the same <inputs> and different <serviceInputs>
- *                                  would be considered as copies of each-other.
- * @return renderer: a smart resource with additional output("name") method
- */
+
+
+
+
+
+
+
+
+
+
+
 create := func(tpl, inputs, ...opts) {
 	return createUniversal(tpl, false, inputs, opts...)
 }
 
-/**
- * Creates ephemeral render template resource.
- *
- * @param tpl: template resource (see assets.importTemplate())
- * @param inputs: a map of inputs to be passed to the template
- * @param serviceInputs: (optional) a map of service inputs to be passed to the template.
- *                                  The split into <inputs> and <serviceInputs> has no
- *                                  effect on ephemeral templates, as they do not
- *                                  have deduplication.
- * @return renderer: a smart resource with additional output("name") method
- */
+
+
+
+
+
+
+
+
+
+
+
 createEphemeral := func(tpl, inputs, ...opts) {
 	return createUniversal(tpl, true, inputs, opts...)
 }

package/dist/tengo/lib/sets.lib.tengo

@@ -1,18 +1,18 @@
-/**
- * Functions for working with sets of elements,
- * in Tengo they are just maps with truthy values.
- * Keys of the map can be only either strings or numbers.
- */
+
+
+
+
+
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
 maps := import("@platforma-sdk/workflow-tengo:maps")
 
-/**
- * Turns a slice to a set of values.
- *
- * @param slice: slice<string|number>
- * @return result: set<string|number>
- */
+
+
+
+
+
+
 fromSlice := func(slice) {
 	set := {}
 	for _, elem in slice {
@@ -21,12 +21,12 @@ fromSlice := func(slice) {
 	return set
 }
 
-/**
- * Convert given set to slice
- *
- * @param set: set - a set of values to be converted to slice
- * @return slice: []<string|number>
- */
+
+
+
+
+
+
 toSlice := func(set) {
 	sl := []
 	for v, _ in set {
@@ -35,24 +35,24 @@ toSlice := func(set) {
 	return sl
 }
 
-/**
- * Checks if a set has an element.
- *
- * @param set a set of any elements
- * @param element any
- * @return found: bool
- */
+
+
+
+
+
+
+
 hasElement := func(set, element) {
 	return !is_undefined(set[element])
 }
 
-/**
- * Adds elements to the set.
- *
- * @param set a set
- * @param elements a slice of elements
- * @return: result: set<string|number> - a changed old set with all old and new elements.
- */
+
+
+
+
+
+
+
 add := func(set, ...elements) {
 	for _, elem in elements {
 		set[elem] = true
@@ -61,24 +61,24 @@ add := func(set, ...elements) {
 	return set
 }
 
-/**
- * Returns a set of key of the map.
- *
- * @param map a map
- * @return result: set<string|number>
- */
+
+
+
+
+
+
 fromMapKeys := func(map) {
 	keys := maps.getKeys(map)
 	return fromSlice(keys)
 }
 
-/**
- * Transforms all keys of the map with a function and returns a new map with these keys.
- *
- * @param set: set
- * @param fn: func(elem): newElem - a function that gets an element and returns a new element.
- * @return new set: map[newElem]any - a transformed set
- */
+
+
+
+
+
+
+
 map := func(set, fn) {
 	return maps.mapKeys(set, fn)
 }