@platforma-sdk/workflow-tengo 1.2.6

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -0,0 +1,244 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+objects := import("@platforma-sdk/workflow-tengo:objects")
+
+_PRE_PROCESS_STEP_SCHEMA := [`or`,
+	{
+		// Replace cell value with the first full RegEx match
+		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/match
+		`type`: `string,regex=regexpExtract`,
+		// String representing ECMAScript RegEx
+		// If there are no or more than one matches - result would become NA
+		`pattern`: `string`
+	},
+	{
+		// Perform JS String.prototype.replace() operation
+		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace
+		`type`: `string,regex=regexpReplace`,
+		// String representing ECMAScript RegEx with at least one capturing group.
+		// If you need to reorder capturing groups - use RegExp matching the whole string
+		// (must start with string begin anchor ^, end with string end anchor $).
+		// Use regex playground https://regexr.com/ to test your ideas.
+		`pattern`: `string`,
+		// Replacement pattern used to construct result string from captured groups
+		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement
+		// Empty string as result would become NA.
+		`replacement`: `string`
+	}]
+
+_SPEC_AXES_SCHEMA := {
+	// Column label from XSV
+	`column`: `string`,
+	// Pre-processing operations to be performed just after reading XSV, before any other operations
+	`preProcess,omitempty`: [_PRE_PROCESS_STEP_SCHEMA],
+	// Should the N/A values (or empty rows) be allowed;
+	// default: false
+	`allowNA,omitempty`: `bool`,
+	// Specification of the axis
+	`spec`: {
+		// Type of the axis value
+		`type`: `string,regex=Int|Long|Float|Double|String|Bytes`,
+		// Name of the axis to be used in spec;
+		// default: column label
+		`name,omitempty`: `string`,
+		// Auxiliary information to the axis name, type and parents to form a unique identifier;
+		// default: empty
+		`domain,omitempty`: { any: `string` },
+		// Any additional information attached to the axis that does not affect its identifier;
+		// default: "pl7.app/label": Column label from XSV
+		`annotations,omitempty`: { any: `string` },
+		// A list of zero-based indices of parent axes in the overall axes specification;
+		// default: empty
+		`parentAxes,omitempty`: [`number`]
+	}
+}
+
+_SPEC_COLUMN_SCHEMA := {
+	// Column label from XSV
+	`column`: `string`,
+	// ID of the column to be used in spec and as a saved filename;
+	// default: column label with all special characters replaced with '_'
+	`id,omitempty`: `string`,
+	// Pre-processing operations to be performed just after reading XSV, before any other operations;
+	// warning: when pre-processing is used, spec valueType must be set
+	`preProcess,omitempty`: [_PRE_PROCESS_STEP_SCHEMA],
+	// Should the N/A values (or empty rows) be allowed;
+	// default: true
+	`allowNA,omitempty`: `bool`,
+	// Specification of the column
+	`spec,omitempty`: {
+		// Type of column values;
+		// default: automatically detected to be one of Long, Double, String
+		`type,omitempty`: `string,regex=Int|Long|Float|Double|String|Bytes`,
+		// Name of the column to be used in spec;
+		// default: column label
+		`name,omitempty`: `string`,
+		// Auxiliary information to the column name, type and parents to form a unique identifier;
+		// default: empty
+		`domain,omitempty`: { any: `string` },
+		// Any additional information attached to the axis that does not affect its identifier;
+		// default: "pl7.app/label": Column label from XSV
+		`annotations,omitempty`: { any: `string` },
+		// A list of zero-based indices of parent axes in the overall axes specification;
+		// default: empty
+		`parentAxes,omitempty`: [ `number` ]
+	}
+}
+
+_SPEC_INDEX_SCHEMA := {
+	// Name of the axis
+	`name`: `string`,
+	// Auxiliary information to the column name, type and parents to form a unique identifier;
+	// default: empty
+	`domain,omitempty`: { any: `string` },
+	// Any additional information attached to the axis that does not affect its identifier;
+	// default: "pl7.app/label": name
+	`annotations,omitempty`: { any: `string` },
+	// A list of zero-based indices of parent axes in the overall axes specification;
+	// default: empty
+	`parentAxes,omitempty`: [`number`]
+}
+
+PFCONV_CFG_SCHEMA := {
+	// Columns would be stored using specified format;
+	// default: Binary
+	`storageFormat,omitempty` : `string,regex=Binary|Json`,
+	// Partitioning key length;
+	// default: 0
+	`partitionKeyLength,omitempty` : `number`,
+	// XSV columns to use as PColumn axes, order of axes in resulting columns
+	// would match the order of spec entries provided here
+	`axes`: [_SPEC_AXES_SCHEMA],
+	// XSV columns to use as PColumn values, each exported individually
+	`columns`: [_SPEC_COLUMN_SCHEMA],
+	// Axis of type Int with XSV row numbers (would be the last one);
+	// default: do not create additional axis with indexes
+	`index,omitempty`: _SPEC_INDEX_SCHEMA,
+	// String representing NA values of types Int, Long, Float, Double
+	// default: NaN
+	`naStr,omitempty`: `string`,
+	// String representing NA values of types String, Bytes
+	// default: null
+	`nullStr,omitempty`: `string`,
+	// Single ASCII character to be used as separator;
+	// default: ',' for .csv, '\t' for .tsv
+	`separator,omitempty`: `string`,
+	// Interpret empty string as NA value
+	// default: true
+	`emptyStrAsNA,omitempty`: `bool`,
+	// Prefix all column names with given string;
+	// default: names would be preserved
+	`columnNamePrefix,omitempty`: `string`,
+	// Resolve duplicates by adding sequential suffixes to column labels;
+	// default: true
+	`allowColumnLabelDuplicates,omitempty`: `bool`,
+	// When columns spec is provided but no such column was found in XSV - create such column
+	// filled with NA values instead of failing;
+	// default: false
+	`allowArtificialColumns,omitempty`: `bool`
+}
+
+
+/**
+ * Get column id from the column spec
+ */
+xsvColumnId := func(c) {
+	id := c.id
+	if is_undefined(id) {
+		id = c.column
+	}
+	return id
+}
+
+/**
+ * Removes all annotations & domain information from spec, to pass it to pfconv
+ */
+purifySpec := func(spec) {
+	newSpec := copy(spec)
+
+	newAxes := []
+	for ax in spec.axes {
+		newAxes = append(newAxes, {
+			column: ax.column,
+			spec: { type: ax.spec.type }
+		})
+	}
+	newSpec.axes = newAxes
+
+	newCols := []
+	for col in spec.columns {
+		newCols = append(newCols, objects.deleteUndefined({
+			column: col.column,
+			id: col.id,
+			allowNA: col.allowNA,
+			spec: { valueType: col.spec.valueType }
+		}))
+	}
+	newSpec.columns = newCols
+
+	validation.assertJsonSchema(newSpec, PFCONV_CFG_SCHEMA)
+
+	return newSpec
+}
+
+/**
+ * Split given pfconv config into:
+ *
+ *  (1) purified cfg with minimal required settings to be passed to the pfconv binary,
+ *      to maximize caching retention when domains and annotations changes
+ *  (2) complete columns specs, that can be directly rendered into BObjectSpecs and
+ *      returned as workflow exports
+ *
+ * This method comes in handy in situations where actual csv -> pframe export is broadcasted
+ * onto multiple objects, and then converges, so that each export operation only requires purified
+ * config, and only data is saved for such operations, and actual specs are added somewhere
+ * upstream.
+ *
+ * @param cfg  full pfconv config object
+ * @param ops  additional options
+ *               {
+ *                 additionalAxesSpec  - array of additional axes spec to prepend to each column spec
+ *               }
+ *
+ * @return {
+ *           purifiedCfg  - purified pfconv config
+ *           columnsSpec   - map of full column specs by ids from the original cfg
+ *         }
+ */
+decomposePfconvCfg := func(cfg, ...ops) {
+	ll.assert(len(ops) <= 1, "too many options for decomposePfconvCfg: %v", len(ops))
+
+	axesSpec := []
+	if len(ops) == 1 && !is_undefined(ops[0].additionalAxesSpec) {
+		axesSpec = ops[0].additionalAxesSpec
+	}
+
+	for axis in cfg.axes {
+		axesSpec = append(axesSpec, axis.spec)
+	}
+
+	columnsSpec := {}
+
+	for column in cfg.columns {
+		id := xsvColumnId(column)
+		ll.assert(is_undefined(columnsSpec[id]), "repeated column spec id in pfconv cfg: %v", id)
+		columnsSpec[id] = column.spec
+		columnsSpec[id].axesSpec = axesSpec
+		// make sure spec contain correct kind value
+		columnsSpec[id].kind = "PColumn"
+	}
+
+	return {
+		purifiedCfg: purifySpec(cfg),
+		columnsSpec: columnsSpec
+	}
+}
+
+export ll.toStrict({
+	PFCONV_CFG_SCHEMA: PFCONV_CFG_SCHEMA,
+
+	xsvColumnId: xsvColumnId,
+	purifySpec: purifySpec,
+	decomposePfconvCfg: decomposePfconvCfg
+})

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -0,0 +1,178 @@
+/**
+ * Library providing xsv <> p-frames conversion utils.
+ */
+
+validation := import("@platforma-sdk/workflow-tengo:validation")
+constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+objects := import("@platforma-sdk/workflow-tengo:objects")
+render := import("@platforma-sdk/workflow-tengo:render")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+json := import("json")
+util := import("@platforma-sdk/workflow-tengo:pframes.util")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+
+pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
+
+/**
+ * Calculate axes spec from pfconv spec
+ */
+getAxesSpec := func(spec) {
+	axes := []
+	for ax in spec.axes {
+		axes = append(axes, ax.spec)
+	}
+	return axes
+}
+
+renderOps := func(defaults, ...ops) {
+	o := { }
+
+	for k, v in defaults {
+		o[k] = v
+	}
+
+	if len(ops) > 0 {
+		for k, v in ops[0] {
+			o[k] = v
+		}
+	}
+	return o
+}
+
+getColumnSpec := func(axesSpec, col) {
+	spec := {
+		kind: constants.KIND_P_COLUMN,
+		axesSpec: axesSpec
+	}
+
+	for k, v in col.spec {
+		spec[k] = v
+	}
+
+	return objects.deleteUndefined(spec)
+}
+
+/**
+ * Imports xsv data into p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
+ * provided by the spec used as map keys). Resulting p-columns will be always single-partitioned at most.
+ *
+ * @param xsvFile: reference - a reference to a file
+ * @param xsvType: string - either csv or tsv
+ * @param spec: object - xsv conversion specification
+ * @param ops: dict - additional params (internal)
+ * @return map: reference - a reference to a map resource storing imported data.
+ */
+importFile := func(xsvFile, xsvType, spec, ...ops) {
+	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
+
+	validation.assertJsonSchema(spec, util.PFCONV_CFG_SCHEMA)
+
+	xsvFileName := "file." + xsvType
+
+	// import p-frame from directory tpl
+	importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
+
+	// spec without any metadata (for caching purposes)
+	pureSpec := util.purifySpec(spec)
+
+	// convert csv to p-frame and read resulting data
+	pfconv := exec.builder().
+		software(pfconvSw).
+		arg("importCsv").
+		arg(xsvFileName).
+		arg("-p").arg("spec.json").
+		arg("-o").arg("out").
+		addFile(xsvFileName, xsvFile).
+		writeFile("spec.json", json.encode(pureSpec)).
+		processWorkdir("pf", importDirTpl, pureSpec).
+		run()
+
+    ops := renderOps({ dataOnly: false }, ops...)
+
+	// p-columns data
+	pf := pfconv.getProcessorResult("pf")
+	if ops.dataOnly {
+		return pf
+	}
+
+	axesSpec := getAxesSpec(spec)
+
+	result := {}
+	for col in spec.columns {
+		id := util.xsvColumnId(col)
+
+		result[id + ".data"] = pf.getFutureInputField(id)
+		result[id + ".spec"] = getColumnSpec(axesSpec, col)
+	}
+
+	return result
+}
+
+/**
+ * Imports a map with xsv files into a p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
+ * provided by the spec used as map keys). Resulting p-columns may be double-partitioned.
+ *
+ * @param xsvFile: reference - a reference to a file
+ * @param xsvType: string - either csv or tsv
+ * @param spec: object - xsv conversion specification
+ * @param ops: object - additional options
+ *               {
+ *                 dataOnly: boolean  - set to true to completely skip creation of specs
+ *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
+ *                                              with "spec" + "data" fields in nested maps
+ *                 additionalAxesSpec: AxisSpec[]  - array of additional axes spec to prepend to
+ *                                                   each column spec
+ *               }
+ * @return map: reference - a reference to a map resource storing imported data.
+ */
+importFileMap := func(xsvMap, xsvType, spec, ...ops) {
+
+	importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-xsv-map")
+
+	ops := renderOps({
+	    dataOnly: false,
+		splitDataAndSpec: false,
+		additionalAxesSpec: []
+	  }, ops...)
+
+	// spec without any metadata (for caching purposes)
+	decomposedSpec := util.decomposePfconvCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
+	pureSpec := decomposedSpec.purifiedCfg
+	columnsSpec := decomposedSpec.columnsSpec
+
+	r := render.createEphemeral(importXsvMapTpl, {
+		xsvMap: xsvMap,
+		xsvType: xsvType,
+		spec: pureSpec,
+		ops: { dataOnly: true }
+	})
+
+	if ops.dataOnly && ops.splitDataAndSpec {
+		ll.panic("Can't use both options dataOnly and splitDataAndSpec at the same time.")
+	}
+
+	result := {}
+	axesSpec := getAxesSpec(spec)
+	for c in spec.columns {
+		id := util.xsvColumnId(c)
+		if ops.dataOnly {
+			result[id] = r.output(id)
+		} else if ops.splitDataAndSpec {
+			result[id] = {
+				data: r.output(id),
+				spec: columnsSpec[id]
+			}
+		} else {
+			result[id + ".data"] = r.output(id)
+			result[id + ".spec"] = columnsSpec[id]
+		}
+	}
+
+	return result
+}
+
+export ll.toStrict({
+	importFile: importFile,
+	importFileMap: importFileMap
+})

package/dist/tengo/lib/render.lib.tengo

@@ -0,0 +1,100 @@
+/*
+ * Create renderer objects to render templates from the code
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+tplutil := import("@platforma-sdk/workflow-tengo:tpl.util")
+constants := import("@platforma-sdk/workflow-tengo:constants")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+tx := import("tx")
+
+
+_SUBSCRIPTION_FIELD := "subscription"
+
+/**
+ * Creates a render template resource.
+ *
+ * @param tpl: template resource (see assets.importTemplate())
+ * @param isEph: whether template is ephemeral
+ * @param inputs: a map of inputs to be passed to the template
+ * @return renderer: a smart resource with additional output("name") method
+ */
+createUniversal := func(tpl, isEph, inputs) {
+
+	// build renderer resource
+	rbuilder := undefined
+	if isEph {
+		rbuilder = smart.ephemeralBuilder(constants.RTYPE_RENDER_EPH_TEMPLATE)
+	}  else {
+		rbuilder = smart.structBuilder(constants.RTYPE_RENDER_TEMPLATE)
+	}
+
+	if isEph {
+		// create internal subscription field
+		rbuilder.createField(_SUBSCRIPTION_FIELD, "OTW")
+	}
+
+	// template input
+	rbuilder.createInputField(tplutil.TEMPLATE_FIELD).set(tpl)
+
+	// inputs input
+	inputsMap := isEph ? smart.ephemeralMapBuilder() : smart.mapBuilder()
+	for name, value in inputs {
+		inputsMap.add(name, value)
+	}
+
+	rbuilder.createInputField(tplutil.INPUTS_MAP_FIELD).set(inputsMap.build())
+	// build
+	renderer := rbuilder.lockInputsAndBuild()
+
+	self := undefined
+	self = ll.toStrict(oop.inherit(renderer, {
+
+		/**
+		 * Returns an output field with a given name
+		 */
+		output: func(name, ...cache) {
+			ll.assert(len(cache) == 0 || is_int(cache[0]), "cache should be a number (milliseconds)")
+			out := renderer.getFutureOutputField(tplutil.TEMPLATE_OUTPUT_PREFIX + name)
+			if len(cache) > 0 && cache[0] > 0 {
+				cache = cache[0]
+				tx.cacheSetToField(out.id, cache)
+			}
+
+			return out
+		}
+	}))
+
+	return self
+}
+
+/**
+ * Creates a render template resource.
+ *
+ * @param tpl: template resource (see assets.importTemplate())
+ * @param inputs: a map of inputs to be passed to the template
+ * @return renderer: a smart resource with additional output("name") method
+ */
+create := func(tpl, inputs) {
+	return createUniversal(tpl, false, inputs)
+}
+
+/**
+ * Creates ephemeral render template resource.
+ *
+ * @param tpl: template resource (see assets.importTemplate())
+ * @param inputs: a map of inputs to be passed to the template
+ * @return renderer: a smart resource with additional output("name") method
+ */
+createEphemeral := func(tpl, inputs) {
+	return createUniversal(tpl, true, inputs)
+}
+
+export ll.toStrict({
+	createUniversal: createUniversal,
+	create: create,
+	createEphemeral: createEphemeral
+})

package/dist/tengo/lib/sets.lib.tengo

@@ -0,0 +1,93 @@
+/**
+ * Functions for working with sets of elements,
+ * in Tengo they are just maps with truthy values.
+ * Keys of the map can be only either strings or numbers.
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+
+/**
+ * Turns a slice to a set of values.
+ *
+ * @param slice: slice<string|number>
+ * @return result: set<string|number>
+ */
+fromSlice := func(slice) {
+	set := {}
+	for _, elem in slice {
+		set[elem] = true
+	}
+	return set
+}
+
+/**
+ * Convert given set to slice
+ *
+ * @param set: set - a set of values to be converted to slice
+ * @return slice: []<string|number>
+ */
+toSlice := func(set) {
+	sl := []
+	for v, _ in set {
+		sl = append(sl, v)
+	}
+	return sl
+}
+
+/**
+ * Checks if a set has an element.
+ *
+ * @param set a set of any elements
+ * @param element any
+ * @return found: bool
+ */
+hasElement := func(set, element) {
+	return !is_undefined(set[element])
+}
+
+/**
+ * Adds elements to the set.
+ *
+ * @param set a set
+ * @param elements a slice of elements
+ * @return: result: set<string|number> - a changed old set with all old and new elements.
+ */
+add := func(set, ...elements) {
+	for _, elem in elements {
+		set[elem] = true
+	}
+
+	return set
+}
+
+/**
+ * Returns a set of key of the map.
+ *
+ * @param map a map
+ * @return result: set<string|number>
+ */
+fromMapKeys := func(map) {
+	keys := maps.getKeys(map)
+	return fromSlice(keys)
+}
+
+/**
+ * Transforms all keys of the map with a function and returns a new map with these keys.
+ *
+ * @param set: set
+ * @param fn: func(elem): newElem - a function that gets an element and returns a new element.
+ * @return new set: map[newElem]any - a transformed set
+ */
+map := func(set, fn) {
+	return maps.mapKeys(set, fn)
+}
+
+export ll.toStrict({
+	fromSlice: fromSlice,
+	toSlice: toSlice,
+	hasElement: hasElement,
+	add: add,
+	fromMapKeys: fromMapKeys,
+	map: map
+})