@platforma-sdk/workflow-tengo 2.8.0 → 2.8.2

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -1,325 +1,190 @@
 ll := import("@platforma-sdk/workflow-tengo:ll")
 validation := import("@platforma-sdk/workflow-tengo:validation")
-constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 objects := import("@platforma-sdk/workflow-tengo:objects")
 
 _PRE_PROCESS_STEP_SCHEMA := [`or`,
-	{
-		// Perform JS String.prototype.replace() operation
-		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace
-		`type`: `string,regex=regexpReplace`,
-		// String representing ECMAScript RegEx with at least one capturing group.
-		// If you need to reorder capturing groups - use RegExp matching the whole string
-		// (must start with string begin anchor ^, end with string end anchor $).
-		// Use regex playground https://regexr.com/ to test your ideas.
-		`pattern`: `string`,
-		// Replacement pattern used to construct result string from captured groups
-		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement
-		// Empty string as result would become NA.
-		`replacement`: `string`
-	},
-	{
-		// Simplified 'regexpReplace' with replacement set to $1.
-		// This means that string is replaced with first capture group value.
-		// Example 1:
-		// - input: 123___abc.xlsx
-		// - pattern: \d+___([a-z]+).xlsx
-		// - result: abc
-		// Example 2:
-		// - input: 123___abc.xlsx
-		// - pattern: (\d+)___([a-z]+).xlsx
-		// - result: 123
-		// Example 3:
-		// - input: 123___abc.xlsx
-		// - pattern: ((\d+)___([a-z]+)).xlsx
-		// - result: 123___abc
-		`type`: `string,regex=regexpExtract`,
-		// String representing ECMAScript RegEx with at least one capturing group.
-		// RegEx must match the entire string, this would be enforced even when ^ and $ are skipped.
-		// If there are no matches - value would be replaced with empty string.
-		// Wrong example:
-		// - input: 123___abc.xlsx
-		// - pattern: (\d+___[a-z]+)
-		// - result: empty string, as .xlsx part is missing in pattern, so pattern was not matched
-		// Correct example:
-		// - input: 123___abc.xlsx
-		// - pattern: (\d+___[a-z]+).xlsx
-		// - result: 123___abc
-		`pattern`: `string`
-	}]
+{
+`type`: `string,regex=regexpReplace`,
+`pattern`: `string`,
+`replacement`: `string`
+},
+{
+`type`: `string,regex=regexpExtract`,
+`pattern`: `string`
+}]
 
 _SPEC_AXES_SCHEMA := {
-	// Column label from XSV
-	`column`: `string`,
-	// Regular expression, if matched - whole row would be skipped;
-	// Performed just after reading XSV before any other operations;
-	// default: no filter
-	`filterOutRegex,?`: `string`,
-	// Pre-processing operations to be performed just after filtering out rows
-	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
-	// Regular expression, if matched - value is considered to be N/A;
-	// default: Int, Long - check that the value is a valid integer number;
-	//          Float, Double - check that the value is a valid float number, not a NaN;
-	//          String - accept anything;
-	`naRegex,?`: `string`,
-	// Should the N/A values (or empty rows) be allowed;
-	// default: false
-	`allowNA,?`: `bool`,
-	// Specification of the axis
-	`spec`: {
-		// Type of axis values
-		`type`: `string,regex=Int|Long|Float|Double|String`,
-		// Name of the axis to be used in spec;
-		// default: column label
-		`name,?`: `string`,
-		// Auxiliary information to the axis name, type and parents to form a unique identifier;
-		// default: empty
-		`domain,?`: { any: `string` },
-		// Any additional information attached to the axis that does not affect its identifier;
-		// default: "pl7.app/label": Column label from XSV
-		`annotations,?`: { any: `string` },
-		// A list of zero-based indices of parent axes in the overall axes specification;
-		// default: empty
-		`parentAxes,?`: [`number`]
-	}
+`column`: `string`,
+`filterOutRegex,?`: `string`,
+`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
+`naRegex,?`: `string`,
+`allowNA,?`: `bool`,
+`spec`: {
+`type`: `string,regex=Int|Long|Float|Double|String`,
+`name,?`: `string`,
+`domain,?`: { any: `string` },
+`annotations,?`: { any: `string` },
+`parentAxes,?`: [`number`]
+}
 }
 
 _SPEC_COLUMN_SCHEMA := {
-	// Column label from XSV
-	`column`: `string`,
-	// Regular expression, if matched - whole row would be skipped;
-	// Performed just after reading XSV before any other operations;
-	// default: no filter
-	`filterOutRegex,?`: `string`,
-	// Pre-processing operations to be performed just after filtering out rows
-	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
-	// Regular expression, if matched - value is considered to be N/A;
-	// default: Int, Long - check that the value is a valid integer number;
-	//          Float, Double - check that the value is a valid float number, not a NaN;
-	//          String - accept anything;
-	`naRegex,?`: `string`,
-	// Should the N/A values (or empty rows) be allowed;
-	// default: true
-	`allowNA,?`: `bool`,
-	// ID of the column to be used in spec and as a saved filename;
-	// default: column label with all special characters replaced with '_'
-	`id,?`: `string`,
-	// Specification of the column
-	`spec`: {
-		// Type of column values
-		`valueType`: `string,regex=Int|Long|Float|Double|String`,
-		// Name of the column to be used in spec;
-		// default: column label
-		`name,?`: `string`,
-		// Auxiliary information to the column name, type and parents to form a unique identifier;
-		// default: empty
-		`domain,?`: { any: `string` },
-		// Any additional information attached to the axis that does not affect its identifier;
-		// default: "pl7.app/label": Column label from XSV
-		`annotations,?`: { any: `string` },
-		// A list of zero-based indices of parent axes in the overall axes specification;
-		// default: empty
-		`parentAxes,?`: [ `number` ]
-	}
+`column`: `string`,
+`filterOutRegex,?`: `string`,
+`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
+`naRegex,?`: `string`,
+`allowNA,?`: `bool`,
+`id,?`: `string`,
+`spec`: {
+`valueType`: `string,regex=Int|Long|Float|Double|String`,
+`name,?`: `string`,
+`domain,?`: { any: `string` },
+`annotations,?`: { any: `string` },
+`parentAxes,?`: [ `number` ]
+}
 }
 
 _SPEC_INDEX_SCHEMA := {
-	// Name of the axis
-	`name`: `string`,
-	// Auxiliary information to the column name, type and parents to form a unique identifier;
-	// default: empty
-	`domain,?`: { any: `string` },
-	// Any additional information attached to the axis that does not affect its identifier;
-	// default: "pl7.app/label": name
-	`annotations,?`: { any: `string` },
-	// A list of zero-based indices of parent axes in the overall axes specification;
-	// default: empty
-	`parentAxes,?`: [`number`]
+`name`: `string`,
+`domain,?`: { any: `string` },
+`annotations,?`: { any: `string` },
+`parentAxes,?`: [`number`]
 }
 
 PFCONV_IMPORT_CFG_SCHEMA := {
-	// Single ASCII character to be used as separator;
-	// default: ',' for .csv, '\t' for .tsv
-	`separator,?`: `string`,
-	// Single ASCII character, if XSV row begins with this character - the row would be skipped;
-	// default: error is thrown if comment lines are found in document
-	`commentLinePrefix,?`: `string`,
-	// Should empty lines be skipped;
-	// default: false
-	`skipEmptyLines,?`: `bool`,
-	// Resolve duplicates by adding sequential suffixes to column labels;
-	// default: true
-	`allowColumnLabelDuplicates,?`: `bool`,
-	// XSV columns to use as PColumn axes, order of axes in resulting columns
-	// would match the order of spec entries provided here
-	`axes`: [_SPEC_AXES_SCHEMA],
-	// Axis of type Long with XSV row numbers (would be the last one);
-	// default: do not create additional axis with indexes
-	`index,?`: _SPEC_INDEX_SCHEMA,
-	// XSV columns to use as PColumn values, each exported individually
-	`columns`: [_SPEC_COLUMN_SCHEMA],
-	// When columns spec is provided but no such column was found in XSV - create such column
-	// filled with NA values instead of failing;
-	// default: false
-	`allowArtificialColumns,?`: `bool`,
-	// Prefix all column names with given string;
-	// default: names would be preserved
-	`columnNamePrefix,?`: `string`,
-	// Columns would be stored using specified format;
-	// default: Binary
-	`storageFormat,?` : `string,regex=Binary|Json`,
-	// Partitioning key length;
-	// default: 0
-	`partitionKeyLength,?` : `number`
+`separator,?`: `string`,
+`commentLinePrefix,?`: `string`,
+`skipEmptyLines,?`: `bool`,
+`allowColumnLabelDuplicates,?`: `bool`,
+`axes`: [_SPEC_AXES_SCHEMA],
+`index,?`: _SPEC_INDEX_SCHEMA,
+`columns`: [_SPEC_COLUMN_SCHEMA],
+`allowArtificialColumns,?`: `bool`,
+`columnNamePrefix,?`: `string`,
+`storageFormat,?` : `string,regex=Binary|Json`,
+`partitionKeyLength,?` : `number`
 }
 
 _COLUMN_FILTER_SCHEMA := {
-	// Match any of the types listed here;
-	// default: type is ignored during matching
-	`type,?`: `string,regex=Int|Long|Float|Double|String|Bytes`,
-	// Match any of the names listed here;
-	// default: name is ignored during matching
-	`name,?`: `string`,
-	// Match requires all the listed domains to have provided values;
-	// default: no domain equality checks are performed during matching
-	`domainValue,?`: { any: `string` },
-	// Match requires all the listed annotations to have provided values;
-	// default: no annotation equality checks are performed during matching
-	`annotationValue,?`: { any: `string` },
-	// Match requires all the listed annotations to match provided regex patterns;
-	// default: no annotation pattern matching is performed during matching
-	`annotationPattern,?`: { any: `string` }
+`type,?`: `string,regex=Int|Long|Float|Double|String|Bytes`,
+`name,?`: `string`,
+`domainValue,?`: { any: `string` },
+`annotationValue,?`: { any: `string` },
+`annotationPattern,?`: { any: `string` }
 }
 
 PFCONV_EXPORT_CFG_SCHEMA := {
-	// String representing NA values of types Int, Long, Float, Double;
-	// default: NaN
-	`naStr,?`: `string`,
-	// String representing NA and missing values of types String, Bytes;
-	// default: null
-	`nullStr,?`: `string`,
-	// Toggle to export or skip axes and columns of type Bytes;
-	// default: false
-	`exportBytes,?`: `bool`,
-	// Single ASCII character to be used as separator;
-	// default: ',' for .csv, '\t' for .tsv
-	`separator,?`: `string`,
-	// Only columns matching the selector are exported;
-	// default: all columns are exported
-	`columnSelector,?`: [_COLUMN_FILTER_SCHEMA],
-	// During export, columns with equal axes specifications would be joined into single table
-	// using the join type specified;
-	// default: Full
-	`joinType,?`: `string,regex=Full|Inner`
+`naStr,?`: `string`,
+`nullStr,?`: `string`,
+`exportBytes,?`: `bool`,
+`separator,?`: `string`,
+`columnSelector,?`: [_COLUMN_FILTER_SCHEMA],
+`joinType,?`: `string,regex=Full|Inner`
 }
 
-/**
- * Get column id from the column spec
- */
+
+
 xsvColumnId := func(c) {
-	id := c.id
-	if is_undefined(id) {
-		id = c.column
-	}
-	return id
+id := c.id
+if is_undefined(id) {
+id = c.column
+}
+return id
 }
 
-/**
- * Removes all annotations & domain information from spec, to pass it to pfconv
- */
+
+
 purifySpec := func(spec) {
-	newSpec := copy(spec)
-
-	newAxes := []
-	for ax in spec.axes {
-		newAxes = append(newAxes, objects.deleteUndefined({
-			column: ax.column,
-			filterOutRegex: ax.filterOutRegex,
-			preProcess: ax.preProcess,
-			naRegex: ax.naRegex,
-			allowNA: ax.allowNA,
-			spec: { type: ax.spec.type }
-		}))
-	}
-	newSpec.axes = newAxes
-
-	newCols := []
-	for col in spec.columns {
-		newCols = append(newCols, objects.deleteUndefined({
-			column: col.column,
-			filterOutRegex: col.filterOutRegex,
-			preProcess: col.preProcess,
-			naRegex: col.naRegex,
-			allowNA: col.allowNA,
-			id: col.id,
-			spec: { valueType: col.spec.valueType }
-		}))
-	}
-	newSpec.columns = newCols
-
-	validation.assertJsonSchema(newSpec, PFCONV_IMPORT_CFG_SCHEMA)
-
-	return newSpec
+newSpec := copy(spec)
+
+newAxes := []
+for ax in spec.axes {
+newAxes = append(newAxes, objects.deleteUndefined({
+column: ax.column,
+filterOutRegex: ax.filterOutRegex,
+preProcess: ax.preProcess,
+naRegex: ax.naRegex,
+allowNA: ax.allowNA,
+spec: { type: ax.spec.type }
+}))
+}
+newSpec.axes = newAxes
+
+newCols := []
+for col in spec.columns {
+newCols = append(newCols, objects.deleteUndefined({
+column: col.column,
+filterOutRegex: col.filterOutRegex,
+preProcess: col.preProcess,
+naRegex: col.naRegex,
+allowNA: col.allowNA,
+id: col.id,
+spec: { valueType: col.spec.valueType }
+}))
+}
+newSpec.columns = newCols
+
+validation.assertJsonSchema(newSpec, PFCONV_IMPORT_CFG_SCHEMA)
+
+return newSpec
 }
 
-/**
- * Split given pfconv config into:
- *
- *  (1) purified cfg with minimal required settings to be passed to the pfconv binary,
- *      to maximize caching retention when domains and annotations changes
- *  (2) complete columns specs, that can be directly rendered into BObjectSpecs and
- *      returned as workflow exports
- *
- * This method comes in handy in situations where actual csv -> pframe export is broadcasted
- * onto multiple objects, and then converges, so that each export operation only requires purified
- * config, and only data is saved for such operations, and actual specs are added somewhere
- * upstream.
- *
- * @param cfg  full pfconv config object
- * @param ops  additional options
- *               {
- *                 additionalAxesSpec  - array of additional axes spec to prepend to each column spec
- *               }
- *
- * @return {
- *           purifiedCfg  - purified pfconv config
- *           columnsSpec   - map of full column specs by ids from the original cfg
- *         }
- */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 decomposePfconvImportCfg := func(cfg, ...ops) {
-	ll.assert(len(ops) <= 1, "too many options for decomposePfconvImportCfg: %v", len(ops))
-
-	axesSpec := []
-	if len(ops) == 1 && !is_undefined(ops[0].additionalAxesSpec) {
-		axesSpec = ops[0].additionalAxesSpec
-	}
-
-	for axis in cfg.axes {
-		axesSpec = append(axesSpec, axis.spec)
-	}
-
-	columnsSpec := {}
-
-	for column in cfg.columns {
-		id := xsvColumnId(column)
-		ll.assert(is_undefined(columnsSpec[id]), "repeated column spec id in pfconv cfg: %v", id)
-		columnsSpec[id] = column.spec
-		columnsSpec[id].axesSpec = axesSpec
-		// make sure spec contain correct kind value
-		columnsSpec[id].kind = "PColumn"
-	}
-
-	return {
-		purifiedCfg: purifySpec(cfg),
-		columnsSpec: columnsSpec
-	}
+ll.assert(len(ops) <= 1, "too many options for decomposePfconvImportCfg: %v", len(ops))
+
+axesSpec := []
+if len(ops) == 1 && !is_undefined(ops[0].additionalAxesSpec) {
+axesSpec = ops[0].additionalAxesSpec
+}
+
+for axis in cfg.axes {
+axesSpec = append(axesSpec, axis.spec)
+}
+
+columnsSpec := {}
+
+for column in cfg.columns {
+id := xsvColumnId(column)
+ll.assert(is_undefined(columnsSpec[id]), "repeated column spec id in pfconv cfg: %v", id)
+columnsSpec[id] = column.spec
+columnsSpec[id].axesSpec = axesSpec
+columnsSpec[id].kind = "PColumn"
+}
+
+return {
+purifiedCfg: purifySpec(cfg),
+columnsSpec: columnsSpec
+}
 }
 
 export ll.toStrict({
-	PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
-	PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
+PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
+PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
 
-	xsvColumnId: xsvColumnId,
-	purifySpec: purifySpec,
-	decomposePfconvImportCfg: decomposePfconvImportCfg
+xsvColumnId: xsvColumnId,
+purifySpec: purifySpec,
+decomposePfconvImportCfg: decomposePfconvImportCfg
 })