@platforma-sdk/workflow-tengo 2.13.6 → 2.15.0

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -5,219 +5,219 @@ slices := import("@platforma-sdk/workflow-tengo:slices")
 
 _PRE_PROCESS_STEP_SCHEMA := [`or`,
 	{
-		// Perform JS String.prototype.replace() operation
-		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace
+
+
 		`type`: `string,regex=regexpReplace`,
-		// String representing ECMAScript RegEx with at least one capturing group.
-		// If you need to reorder capturing groups - use RegExp matching the whole string
-		// (must start with string begin anchor ^, end with string end anchor $).
-		// Use regex playground https://regexr.com/ to test your ideas.
+
+
+
+
 		`pattern`: `string`,
-		// Replacement pattern used to construct result string from captured groups
-		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement
-		// Empty string as result would become NA.
+
+
+
 		`replacement`: `string`
 	},
 	{
-		// Simplified 'regexpReplace' with replacement set to $1.
-		// This means that string is replaced with first capture group value.
-		// Example 1:
-		// - input: 123___abc.xlsx
-		// - pattern: \d+___([a-z]+).xlsx
-		// - result: abc
-		// Example 2:
-		// - input: 123___abc.xlsx
-		// - pattern: (\d+)___([a-z]+).xlsx
-		// - result: 123
-		// Example 3:
-		// - input: 123___abc.xlsx
-		// - pattern: ((\d+)___([a-z]+)).xlsx
-		// - result: 123___abc
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 		`type`: `string,regex=regexpExtract`,
-		// String representing ECMAScript RegEx with at least one capturing group.
-		// RegEx must match the entire string, this would be enforced even when ^ and $ are skipped.
-		// If there are no matches - value would be replaced with empty string.
-		// Wrong example:
-		// - input: 123___abc.xlsx
-		// - pattern: (\d+___[a-z]+)
-		// - result: empty string, as .xlsx part is missing in pattern, so pattern was not matched
-		// Correct example:
-		// - input: 123___abc.xlsx
-		// - pattern: (\d+___[a-z]+).xlsx
-		// - result: 123___abc
+
+
+
+
+
+
+
+
+
+
+
 		`pattern`: `string`
 	}]
 
 _SPEC_AXES_SCHEMA := {
-	// Column label from XSV
+
 	`column`: `string`,
-	// Regular expression, if matched - whole row would be skipped;
-	// Performed just after reading XSV before any other operations;
-	// default: no filter
+
+
+
 	`filterOutRegex,?`: `string`,
-	// Pre-processing operations to be performed just after filtering out rows
+
 	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
-	// Regular expression, if matched - value is considered to be N/A;
-	// default: Int, Long - check that the value is a valid integer number;
-	//          Float, Double - check that the value is a valid float number, not a NaN;
-	//          String - accept anything;
+
+
+
+
 	`naRegex,?`: `string`,
-	// Should the N/A values (or empty rows) be allowed; When set to false,
-	// an error would be thrown on N/A value (or empty row) encountering;
-	// default: false
+
+
+
 	`allowNA,?`: `bool`,
-	// Specification of the axis
+
 	`spec`: {
-		// Type of axis values
+
 		`type`: `string,regex=Int|Long|Float|Double|String`,
-		// Name of the axis to be used in spec;
-		// default: column label
+
+
 		`name,?`: `string`,
-		// Auxiliary information to the axis name, type and parents to form a unique identifier;
-		// default: empty
+
+
 		`domain,?`: { any: `string` },
-		// Any additional information attached to the axis that does not affect its identifier;
-		// default: "pl7.app/label": Column label from XSV
+
+
 		`annotations,?`: { any: `string` },
-		// A list of zero-based indices of parent axes in the overall axes specification;
-		// default: empty
+
+
 		`parentAxes,?`: [`number`]
 	}
 }
 
 _SPEC_COLUMN_SCHEMA := {
-	// Column label from XSV
+
 	`column`: `string`,
-	// Regular expression, if matched - whole row would be skipped;
-	// Performed just after reading XSV before any other operations;
-	// default: no filter
+
+
+
 	`filterOutRegex,?`: `string`,
-	// Pre-processing operations to be performed just after filtering out rows
+
 	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
-	// Regular expression, if matched - value is considered to be N/A;
-	// default: Int, Long - check that the value is a valid integer number;
-	//          Float, Double - check that the value is a valid float number, not a NaN;
-	//          String - accept anything;
+
+
+
+
 	`naRegex,?`: `string`,
-	// Should the N/A values (or empty rows) be allowed; When set to false,
-	// an error would be thrown on N/A value (or empty row) encountering;
-	// default: true
+
+
+
 	`allowNA,?`: `bool`,
-	// ID of the column to be used in spec and as a saved filename;
-	// default: column label with all special characters replaced with '_'
+
+
 	`id,?`: `string`,
-	// Specification of the column
+
 	`spec`: {
-		// Type of column values
+
 		`valueType`: `string,regex=Int|Long|Float|Double|String`,
-		// Name of the column to be used in spec;
-		// default: column label
+
+
 		`name,?`: `string`,
-		// Auxiliary information to the column name, type and parents to form a unique identifier;
-		// default: empty
+
+
 		`domain,?`: { any: `string` },
-		// Any additional information attached to the axis that does not affect its identifier;
-		// default: "pl7.app/label": Column label from XSV
+
+
 		`annotations,?`: { any: `string` },
-		// A list of zero-based indices of parent axes in the overall axes specification;
-		// default: empty
+
+
 		`parentAxes,?`: [ `number` ]
 	}
 }
 
 _SPEC_INDEX_SCHEMA := {
-	// Name of the axis
+
 	`name`: `string`,
-	// Auxiliary information to the column name, type and parents to form a unique identifier;
-	// default: empty
+
+
 	`domain,?`: { any: `string` },
-	// Any additional information attached to the axis that does not affect its identifier;
-	// default: "pl7.app/label": name
+
+
 	`annotations,?`: { any: `string` },
-	// A list of zero-based indices of parent axes in the overall axes specification;
-	// default: empty
+
+
 	`parentAxes,?`: [`number`]
 }
 
 PFCONV_IMPORT_CFG_SCHEMA := {
-	// Single ASCII character to be used as separator;
-	// default: ',' for .csv, '\t' for .tsv
+
+
 	`separator,?`: `string`,
-	// Single ASCII character, if XSV row begins with this character - the row would be skipped;
-	// default: error is thrown if comment lines are found in document
+
+
 	`commentLinePrefix,?`: `string`,
-	// Should empty lines be skipped;
-	// default: false
+
+
 	`skipEmptyLines,?`: `bool`,
-	// Resolve duplicates by adding sequential suffixes to column labels;
-	// default: true
+
+
 	`allowColumnLabelDuplicates,?`: `bool`,
-	// XSV columns to use as PColumn axes, order of axes in resulting columns
-	// would match the order of spec entries provided here
+
+
 	`axes`: [_SPEC_AXES_SCHEMA],
-	// Axis of type Long with XSV row numbers (would be the last one);
-	// default: do not create additional axis with indexes
+
+
 	`index,?`: _SPEC_INDEX_SCHEMA,
-	// XSV columns to use as PColumn values, each exported individually
+
 	`columns`: [_SPEC_COLUMN_SCHEMA],
-	// When columns spec is provided but no such column was found in XSV - create such column
-	// filled with NA values instead of failing;
-	// default: false
+
+
+
 	`allowArtificialColumns,?`: `bool`,
-	// Prefix all column names with given string;
-	// default: names would be preserved
+
+
 	`columnNamePrefix,?`: `string`,
-	// Columns would be stored using specified format;
-	// default: Binary
+
+
 	`storageFormat,?` : `string,regex=Binary|Json`,
-	// Partitioning key length;
-	// default: 0
+
+
 	`partitionKeyLength,?` : `number`
 }
 
 _COLUMN_FILTER_SCHEMA := {
-	// Match any of the types listed here;
-	// default: type is ignored during matching
+
+
 	`type,?`: `string,regex=Int|Long|Float|Double|String|Bytes`,
-	// Match any of the names listed here;
-	// default: name is ignored during matching
+
+
 	`name,?`: `string`,
-	// Match requires all the listed domains to have provided values;
-	// default: no domain equality checks are performed during matching
+
+
 	`domainValue,?`: { any: `string` },
-	// Match requires all the listed annotations to have provided values;
-	// default: no annotation equality checks are performed during matching
+
+
 	`annotationValue,?`: { any: `string` },
-	// Match requires all the listed annotations to match provided regex patterns;
-	// default: no annotation pattern matching is performed during matching
+
+
 	`annotationPattern,?`: { any: `string` }
 }
 
 PFCONV_EXPORT_CFG_SCHEMA := {
-	// String representing NA values of types Int, Long, Float, Double;
-	// default: NaN
+
+
 	`naStr,?`: `string`,
-	// String representing NA and missing values of types String, Bytes;
-	// default: null
+
+
 	`nullStr,?`: `string`,
-	// Toggle to export or skip axes and columns of type Bytes;
-	// default: false
+
+
 	`exportBytes,?`: `bool`,
-	// Single ASCII character to be used as separator;
-	// default: ',' for .csv, '\t' for .tsv
+
+
 	`separator,?`: `string`,
-	// Only columns matching the selector are exported;
-	// default: all columns are exported
+
+
 	`columnSelector,?`: [_COLUMN_FILTER_SCHEMA],
-	// During export, columns with equal axes specifications would be joined into single table
-	// using the join type specified;
-	// default: Full
+
+
+
 	`joinType,?`: `string,regex=Full|Inner`
 }
 
-/**
- * Get column id from the column spec
- */
+
+
+
 xsvColumnId := func(c) {
 	id := c.id
 	if is_undefined(id) {
@@ -226,9 +226,9 @@ xsvColumnId := func(c) {
 	return id
 }
 
-/**
- * Removes all annotations & domain information from spec, to pass it to pfconv
- */
+
+
+
 purifySpec := func(spec) {
 	newSpec := copy(spec)
 
@@ -264,31 +264,31 @@ purifySpec := func(spec) {
 	return newSpec
 }
 
-/**
- * Split given pfconv config into:
- *
- *  (1) purified cfg with minimal required settings to be passed to the pfconv binary,
- *      to maximize caching retention when domains and annotations changes
- *  (2) complete columns specs, that can be directly rendered into BObjectSpecs and
- *      returned as workflow exports
- *
- * This method comes in handy in situations where actual csv -> pframe export is broadcasted
- * onto multiple objects, and then converges, so that each export operation only requires purified
- * config, and only data is saved for such operations, and actual specs are added somewhere
- * upstream.
- *
- * @param cfg  full pfconv config object
- * @param ops  additional options
- *               {
- *                 additionalAxesSpec  - array of additional axes spec to prepend to each column spec
- *               }
- *
- * @return {
- *           purifiedCfg  - purified pfconv config
- *           columnsSpec   - map of full column specs by ids from the original cfg
- *         }
- */
- // @TODO: rename to decomposeXsvImport to align with the naming in the rest xsv libs
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 decomposePfconvImportCfg := func(cfg, ...ops) {
 	ll.assert(len(ops) <= 1, "too many options for decomposePfconvImportCfg: %v", len(ops))
 
@@ -308,7 +308,7 @@ decomposePfconvImportCfg := func(cfg, ...ops) {
 		ll.assert(is_undefined(columnsSpec[id]), "repeated column spec id in pfconv cfg: %v", id)
 		columnsSpec[id] = column.spec
 		columnsSpec[id].axesSpec = axesSpec
-		// make sure spec contain correct kind value
+
 		columnsSpec[id].kind = "PColumn"
 	}
 
@@ -320,29 +320,29 @@ decomposePfconvImportCfg := func(cfg, ...ops) {
 
 EMPTY_JSON_KEY := "[]" // string(json.encode([]))
 
-/**
- * Calculates group indices from aggregation indices by removing aggregation indices from the full set of indices.
- * If no aggregation indices are provided, returns all indices.
- *
- * @param numberOfAxes: number - Total number of axes in the input
- * @param aggregationIndices: number[]|undefined - Indices to aggregate over. If undefined, returns all indices.
- *
- * @returns number[] - Array of remaining indices after removing aggregation indices, in ascending order.
- *                     If aggregationIndices is undefined, returns array [0..numberOfAxes-1]
- */
+
+
+
+
+
+
+
+
+
+
 calculateGroupAxesIndices := func(numberOfAxes, aggregationIndices) {
 	groupIndices := []
 	for i:=0; i<numberOfAxes; i++ {
 		groupIndices = append(groupIndices, i)
 	}
 
-	// groupIndices at this point is [0, 1, 2, ... numberOfAxes-1]
+
 
 	if is_undefined(aggregationIndices) {
 		return groupIndices
 	}
 
-	// copy and sort to prevent mutation of original array
+
 	aggregationIndices = slices.quickSort(aggregationIndices)
 
 	for j:=len(aggregationIndices)-1; j>=0; j-- {

package/dist/tengo/lib/pframes.xsv-import-file.lib.tengo

@@ -0,0 +1,160 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+exec := import("@platforma-sdk/workflow-tengo:exec")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+objects := import("@platforma-sdk/workflow-tengo:objects")
+util := import("@platforma-sdk/workflow-tengo:pframes.util")
+json := import("json")
+constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+
+transposeXsvSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:table-converter")
+pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
+
+renderOps := func(defaults, ...ops) {
+	o := { }
+
+	for k, v in defaults {
+		o[k] = v
+	}
+
+	if len(ops) > 0 {
+		for k, v in ops[0] {
+			o[k] = v
+		}
+	}
+	return o
+}
+
+getAxesSpec := func(spec) {
+	axes := []
+	for ax in spec.axes {
+		axes = append(axes, ax.spec)
+	}
+	return axes
+}
+
+getColumnSpec := func(axesSpec, col) {
+	spec := {
+		kind: constants.KIND_P_COLUMN,
+		axesSpec: axesSpec
+	}
+
+	for k, v in col.spec {
+		spec[k] = v
+	}
+
+	return objects.deleteUndefined(spec)
+}
+
+
+
+
+importFile := func(xsvFile, xsvType, spec, ...ops) {
+	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types, found: " + xsvType)
+
+	validation.assertType(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
+
+	xsvFileName := "file." + xsvType
+
+	ops := renderOps({ dataOnly: false, splitDataAndSpec: false }, ops...)
+
+	if ops.transpose != undefined {
+
+		tops := ops.transpose
+
+		transposedFileName := "fileTransposed." + xsvType
+
+
+		transposeXsv := exec.builder().
+			software(transposeXsvSw).
+			printErrStreamToStdout().
+			inLightQueue()
+
+		if (tops.separator != undefined) {
+			transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
+		}
+		if (tops.pAxisIdx != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
+		}
+		if (tops.pAxisName != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
+		}
+		if (tops.pAxisSearch != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
+		}
+		if (tops.pAxisNameOverride != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
+		}
+		if (tops.sAxisSearch != undefined) {
+			transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
+		}
+		if (tops.sAxisName != undefined) {
+			transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)
+		}
+		if (tops.valueName != undefined) {
+			transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)
+		}
+		if (tops.separatorOverride != undefined) {
+			transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)
+		}
+
+		transposeXsv = transposeXsv.
+			arg(xsvFileName).
+			arg(transposedFileName).
+			addFile(xsvFileName, xsvFile).
+			saveFile(transposedFileName).
+			run()
+
+		xsvFile = transposeXsv.getFile(transposedFileName)
+	}
+
+
+	importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
+
+
+	pureSpec := util.purifySpec(spec)
+
+
+	pfconv := exec.builder().
+		software(pfconvSw).
+		printErrStreamToStdout().
+		inLightQueue().
+		arg("importCsv").
+		arg(xsvFileName).
+		arg("-p").arg("spec.json").
+		arg("-o").arg("out").
+		addFile(xsvFileName, xsvFile).
+		writeFile("spec.json", json.encode(pureSpec)).
+		processWorkdir("pf", importDirTpl, pureSpec).
+		run()
+
+
+	pf := pfconv.getProcessorResult("pf")
+	if ops.dataOnly {
+
+		return pf
+	}
+
+	axesSpec := getAxesSpec(spec)
+
+	result := {}
+	for col in spec.columns {
+		id := util.xsvColumnId(col)
+
+		if ops.splitDataAndSpec {
+			result[id] = {
+				data: pf.getFutureInputField(id),
+				spec: getColumnSpec(axesSpec, col)
+			}
+		} else {
+			result[id + ".data"] = pf.getFutureInputField(id)
+			result[id + ".spec"] = getColumnSpec(axesSpec, col)
+		}
+	}
+
+	return result
+}
+
+export ll.toStrict({
+	importFile: importFile
+})