@platforma-sdk/workflow-tengo 2.9.5 → 2.9.7

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -3,322 +3,322 @@ validation := import("@platforma-sdk/workflow-tengo:validation")
 objects := import("@platforma-sdk/workflow-tengo:objects")
 
 _PRE_PROCESS_STEP_SCHEMA := [`or`,
-{
-
-
-`type`: `string,regex=regexpReplace`,
-
-
-
-
-`pattern`: `string`,
-
-
-
-`replacement`: `string`
-},
-{
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-`type`: `string,regex=regexpExtract`,
-
-
-
-
-
-
-
-
-
-
-
-`pattern`: `string`
-}]
+	{
+		// Perform JS String.prototype.replace() operation
+		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace
+		`type`: `string,regex=regexpReplace`,
+		// String representing ECMAScript RegEx with at least one capturing group.
+		// If you need to reorder capturing groups - use RegExp matching the whole string
+		// (must start with string begin anchor ^, end with string end anchor $).
+		// Use regex playground https://regexr.com/ to test your ideas.
+		`pattern`: `string`,
+		// Replacement pattern used to construct result string from captured groups
+		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement
+		// Empty string as result would become NA.
+		`replacement`: `string`
+	},
+	{
+		// Simplified 'regexpReplace' with replacement set to $1.
+		// This means that string is replaced with first capture group value.
+		// Example 1:
+		// - input: 123___abc.xlsx
+		// - pattern: \d+___([a-z]+).xlsx
+		// - result: abc
+		// Example 2:
+		// - input: 123___abc.xlsx
+		// - pattern: (\d+)___([a-z]+).xlsx
+		// - result: 123
+		// Example 3:
+		// - input: 123___abc.xlsx
+		// - pattern: ((\d+)___([a-z]+)).xlsx
+		// - result: 123___abc
+		`type`: `string,regex=regexpExtract`,
+		// String representing ECMAScript RegEx with at least one capturing group.
+		// RegEx must match the entire string, this would be enforced even when ^ and $ are skipped.
+		// If there are no matches - value would be replaced with empty string.
+		// Wrong example:
+		// - input: 123___abc.xlsx
+		// - pattern: (\d+___[a-z]+)
+		// - result: empty string, as .xlsx part is missing in pattern, so pattern was not matched
+		// Correct example:
+		// - input: 123___abc.xlsx
+		// - pattern: (\d+___[a-z]+).xlsx
+		// - result: 123___abc
+		`pattern`: `string`
+	}]
 
 _SPEC_AXES_SCHEMA := {
-
-`column`: `string`,
-
-
-
-`filterOutRegex,?`: `string`,
-
-`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
-
-
-
-
-`naRegex,?`: `string`,
-
-
-`allowNA,?`: `bool`,
-
-`spec`: {
-
-`type`: `string,regex=Int|Long|Float|Double|String`,
-
-
-`name,?`: `string`,
-
-
-`domain,?`: { any: `string` },
-
-
-`annotations,?`: { any: `string` },
-
-
-`parentAxes,?`: [`number`]
-}
+	// Column label from XSV
+	`column`: `string`,
+	// Regular expression, if matched - whole row would be skipped;
+	// Performed just after reading XSV before any other operations;
+	// default: no filter
+	`filterOutRegex,?`: `string`,
+	// Pre-processing operations to be performed just after filtering out rows
+	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
+	// Regular expression, if matched - value is considered to be N/A;
+	// default: Int, Long - check that the value is a valid integer number;
+	//          Float, Double - check that the value is a valid float number, not a NaN;
+	//          String - accept anything;
+	`naRegex,?`: `string`,
+	// Should the N/A values (or empty rows) be allowed;
+	// default: false
+	`allowNA,?`: `bool`,
+	// Specification of the axis
+	`spec`: {
+		// Type of axis values
+		`type`: `string,regex=Int|Long|Float|Double|String`,
+		// Name of the axis to be used in spec;
+		// default: column label
+		`name,?`: `string`,
+		// Auxiliary information to the axis name, type and parents to form a unique identifier;
+		// default: empty
+		`domain,?`: { any: `string` },
+		// Any additional information attached to the axis that does not affect its identifier;
+		// default: "pl7.app/label": Column label from XSV
+		`annotations,?`: { any: `string` },
+		// A list of zero-based indices of parent axes in the overall axes specification;
+		// default: empty
+		`parentAxes,?`: [`number`]
+	}
 }
 
 _SPEC_COLUMN_SCHEMA := {
-
-`column`: `string`,
-
-
-
-`filterOutRegex,?`: `string`,
-
-`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
-
-
-
-
-`naRegex,?`: `string`,
-
-
-`allowNA,?`: `bool`,
-
-
-`id,?`: `string`,
-
-`spec`: {
-
-`valueType`: `string,regex=Int|Long|Float|Double|String`,
-
-
-`name,?`: `string`,
-
-
-`domain,?`: { any: `string` },
-
-
-`annotations,?`: { any: `string` },
-
-
-`parentAxes,?`: [ `number` ]
-}
+	// Column label from XSV
+	`column`: `string`,
+	// Regular expression, if matched - whole row would be skipped;
+	// Performed just after reading XSV before any other operations;
+	// default: no filter
+	`filterOutRegex,?`: `string`,
+	// Pre-processing operations to be performed just after filtering out rows
+	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
+	// Regular expression, if matched - value is considered to be N/A;
+	// default: Int, Long - check that the value is a valid integer number;
+	//          Float, Double - check that the value is a valid float number, not a NaN;
+	//          String - accept anything;
+	`naRegex,?`: `string`,
+	// Should the N/A values (or empty rows) be allowed;
+	// default: true
+	`allowNA,?`: `bool`,
+	// ID of the column to be used in spec and as a saved filename;
+	// default: column label with all special characters replaced with '_'
+	`id,?`: `string`,
+	// Specification of the column
+	`spec`: {
+		// Type of column values
+		`valueType`: `string,regex=Int|Long|Float|Double|String`,
+		// Name of the column to be used in spec;
+		// default: column label
+		`name,?`: `string`,
+		// Auxiliary information to the column name, type and parents to form a unique identifier;
+		// default: empty
+		`domain,?`: { any: `string` },
+		// Any additional information attached to the axis that does not affect its identifier;
+		// default: "pl7.app/label": Column label from XSV
+		`annotations,?`: { any: `string` },
+		// A list of zero-based indices of parent axes in the overall axes specification;
+		// default: empty
+		`parentAxes,?`: [ `number` ]
+	}
 }
 
 _SPEC_INDEX_SCHEMA := {
-
-`name`: `string`,
-
-
-`domain,?`: { any: `string` },
-
-
-`annotations,?`: { any: `string` },
-
-
-`parentAxes,?`: [`number`]
+	// Name of the axis
+	`name`: `string`,
+	// Auxiliary information to the column name, type and parents to form a unique identifier;
+	// default: empty
+	`domain,?`: { any: `string` },
+	// Any additional information attached to the axis that does not affect its identifier;
+	// default: "pl7.app/label": name
+	`annotations,?`: { any: `string` },
+	// A list of zero-based indices of parent axes in the overall axes specification;
+	// default: empty
+	`parentAxes,?`: [`number`]
 }
 
 PFCONV_IMPORT_CFG_SCHEMA := {
-
-
-`separator,?`: `string`,
-
-
-`commentLinePrefix,?`: `string`,
-
-
-`skipEmptyLines,?`: `bool`,
-
-
-`allowColumnLabelDuplicates,?`: `bool`,
-
-
-`axes`: [_SPEC_AXES_SCHEMA],
-
-
-`index,?`: _SPEC_INDEX_SCHEMA,
-
-`columns`: [_SPEC_COLUMN_SCHEMA],
-
-
-
-`allowArtificialColumns,?`: `bool`,
-
-
-`columnNamePrefix,?`: `string`,
-
-
-`storageFormat,?` : `string,regex=Binary|Json`,
-
-
-`partitionKeyLength,?` : `number`
+	// Single ASCII character to be used as separator;
+	// default: ',' for .csv, '\t' for .tsv
+	`separator,?`: `string`,
+	// Single ASCII character, if XSV row begins with this character - the row would be skipped;
+	// default: error is thrown if comment lines are found in document
+	`commentLinePrefix,?`: `string`,
+	// Should empty lines be skipped;
+	// default: false
+	`skipEmptyLines,?`: `bool`,
+	// Resolve duplicates by adding sequential suffixes to column labels;
+	// default: true
+	`allowColumnLabelDuplicates,?`: `bool`,
+	// XSV columns to use as PColumn axes, order of axes in resulting columns
+	// would match the order of spec entries provided here
+	`axes`: [_SPEC_AXES_SCHEMA],
+	// Axis of type Long with XSV row numbers (would be the last one);
+	// default: do not create additional axis with indexes
+	`index,?`: _SPEC_INDEX_SCHEMA,
+	// XSV columns to use as PColumn values, each exported individually
+	`columns`: [_SPEC_COLUMN_SCHEMA],
+	// When columns spec is provided but no such column was found in XSV - create such column
+	// filled with NA values instead of failing;
+	// default: false
+	`allowArtificialColumns,?`: `bool`,
+	// Prefix all column names with given string;
+	// default: names would be preserved
+	`columnNamePrefix,?`: `string`,
+	// Columns would be stored using specified format;
+	// default: Binary
+	`storageFormat,?` : `string,regex=Binary|Json`,
+	// Partitioning key length;
+	// default: 0
+	`partitionKeyLength,?` : `number`
 }
 
 _COLUMN_FILTER_SCHEMA := {
-
-
-`type,?`: `string,regex=Int|Long|Float|Double|String|Bytes`,
-
-
-`name,?`: `string`,
-
-
-`domainValue,?`: { any: `string` },
-
-
-`annotationValue,?`: { any: `string` },
-
-
-`annotationPattern,?`: { any: `string` }
+	// Match any of the types listed here;
+	// default: type is ignored during matching
+	`type,?`: `string,regex=Int|Long|Float|Double|String|Bytes`,
+	// Match any of the names listed here;
+	// default: name is ignored during matching
+	`name,?`: `string`,
+	// Match requires all the listed domains to have provided values;
+	// default: no domain equality checks are performed during matching
+	`domainValue,?`: { any: `string` },
+	// Match requires all the listed annotations to have provided values;
+	// default: no annotation equality checks are performed during matching
+	`annotationValue,?`: { any: `string` },
+	// Match requires all the listed annotations to match provided regex patterns;
+	// default: no annotation pattern matching is performed during matching
+	`annotationPattern,?`: { any: `string` }
 }
 
 PFCONV_EXPORT_CFG_SCHEMA := {
-
-
-`naStr,?`: `string`,
-
-
-`nullStr,?`: `string`,
-
-
-`exportBytes,?`: `bool`,
-
-
-`separator,?`: `string`,
-
-
-`columnSelector,?`: [_COLUMN_FILTER_SCHEMA],
-
-
-
-`joinType,?`: `string,regex=Full|Inner`
+	// String representing NA values of types Int, Long, Float, Double;
+	// default: NaN
+	`naStr,?`: `string`,
+	// String representing NA and missing values of types String, Bytes;
+	// default: null
+	`nullStr,?`: `string`,
+	// Toggle to export or skip axes and columns of type Bytes;
+	// default: false
+	`exportBytes,?`: `bool`,
+	// Single ASCII character to be used as separator;
+	// default: ',' for .csv, '\t' for .tsv
+	`separator,?`: `string`,
+	// Only columns matching the selector are exported;
+	// default: all columns are exported
+	`columnSelector,?`: [_COLUMN_FILTER_SCHEMA],
+	// During export, columns with equal axes specifications would be joined into single table
+	// using the join type specified;
+	// default: Full
+	`joinType,?`: `string,regex=Full|Inner`
 }
 
-
-
-
+/**
+ * Get column id from the column spec
+ */
 xsvColumnId := func(c) {
-id := c.id
-if is_undefined(id) {
-id = c.column
+	id := c.id
+	if is_undefined(id) {
+		id = c.column
+	}
+	return id
 }
-return id
-}
-
-
-
 
+/**
+ * Removes all annotations & domain information from spec, to pass it to pfconv
+ */
 purifySpec := func(spec) {
-newSpec := copy(spec)
-
-newAxes := []
-for ax in spec.axes {
-newAxes = append(newAxes, objects.deleteUndefined({
-column: ax.column,
-filterOutRegex: ax.filterOutRegex,
-preProcess: ax.preProcess,
-naRegex: ax.naRegex,
-allowNA: ax.allowNA,
-spec: { type: ax.spec.type }
-}))
+	newSpec := copy(spec)
+
+	newAxes := []
+	for ax in spec.axes {
+		newAxes = append(newAxes, objects.deleteUndefined({
+			column: ax.column,
+			filterOutRegex: ax.filterOutRegex,
+			preProcess: ax.preProcess,
+			naRegex: ax.naRegex,
+			allowNA: ax.allowNA,
+			spec: { type: ax.spec.type }
+		}))
+	}
+	newSpec.axes = newAxes
+
+	newCols := []
+	for col in spec.columns {
+		newCols = append(newCols, objects.deleteUndefined({
+			column: col.column,
+			filterOutRegex: col.filterOutRegex,
+			preProcess: col.preProcess,
+			naRegex: col.naRegex,
+			allowNA: col.allowNA,
+			id: col.id,
+			spec: { valueType: col.spec.valueType }
+		}))
+	}
+	newSpec.columns = newCols
+
+	validation.assertJsonSchema(newSpec, PFCONV_IMPORT_CFG_SCHEMA)
+
+	return newSpec
 }
-newSpec.axes = newAxes
-
-newCols := []
-for col in spec.columns {
-newCols = append(newCols, objects.deleteUndefined({
-column: col.column,
-filterOutRegex: col.filterOutRegex,
-preProcess: col.preProcess,
-naRegex: col.naRegex,
-allowNA: col.allowNA,
-id: col.id,
-spec: { valueType: col.spec.valueType }
-}))
-}
-newSpec.columns = newCols
-
-validation.assertJsonSchema(newSpec, PFCONV_IMPORT_CFG_SCHEMA)
-
-return newSpec
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 
+/**
+ * Split given pfconv config into:
+ *
+ *  (1) purified cfg with minimal required settings to be passed to the pfconv binary,
+ *      to maximize caching retention when domains and annotations changes
+ *  (2) complete columns specs, that can be directly rendered into BObjectSpecs and
+ *      returned as workflow exports
+ *
+ * This method comes in handy in situations where actual csv -> pframe export is broadcasted
+ * onto multiple objects, and then converges, so that each export operation only requires purified
+ * config, and only data is saved for such operations, and actual specs are added somewhere
+ * upstream.
+ *
+ * @param cfg  full pfconv config object
+ * @param ops  additional options
+ *               {
+ *                 additionalAxesSpec  - array of additional axes spec to prepend to each column spec
+ *               }
+ *
+ * @return {
+ *           purifiedCfg  - purified pfconv config
+ *           columnsSpec   - map of full column specs by ids from the original cfg
+ *         }
+ */
 decomposePfconvImportCfg := func(cfg, ...ops) {
-ll.assert(len(ops) <= 1, "too many options for decomposePfconvImportCfg: %v", len(ops))
-
-axesSpec := []
-if len(ops) == 1 && !is_undefined(ops[0].additionalAxesSpec) {
-axesSpec = ops[0].additionalAxesSpec
-}
-
-for axis in cfg.axes {
-axesSpec = append(axesSpec, axis.spec)
-}
-
-columnsSpec := {}
-
-for column in cfg.columns {
-id := xsvColumnId(column)
-ll.assert(is_undefined(columnsSpec[id]), "repeated column spec id in pfconv cfg: %v", id)
-columnsSpec[id] = column.spec
-columnsSpec[id].axesSpec = axesSpec
-
-columnsSpec[id].kind = "PColumn"
-}
-
-return {
-purifiedCfg: purifySpec(cfg),
-columnsSpec: columnsSpec
-}
+	ll.assert(len(ops) <= 1, "too many options for decomposePfconvImportCfg: %v", len(ops))
+
+	axesSpec := []
+	if len(ops) == 1 && !is_undefined(ops[0].additionalAxesSpec) {
+		axesSpec = ops[0].additionalAxesSpec
+	}
+
+	for axis in cfg.axes {
+		axesSpec = append(axesSpec, axis.spec)
+	}
+
+	columnsSpec := {}
+
+	for column in cfg.columns {
+		id := xsvColumnId(column)
+		ll.assert(is_undefined(columnsSpec[id]), "repeated column spec id in pfconv cfg: %v", id)
+		columnsSpec[id] = column.spec
+		columnsSpec[id].axesSpec = axesSpec
+		// make sure spec contain correct kind value
+		columnsSpec[id].kind = "PColumn"
+	}
+
+	return {
+		purifiedCfg: purifySpec(cfg),
+		columnsSpec: columnsSpec
+	}
 }
 
 export ll.toStrict({
-PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
-PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
+	PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
+	PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
 
-xsvColumnId: xsvColumnId,
-purifySpec: purifySpec,
-decomposePfconvImportCfg: decomposePfconvImportCfg
+	xsvColumnId: xsvColumnId,
+	purifySpec: purifySpec,
+	decomposePfconvImportCfg: decomposePfconvImportCfg
 })