@platforma-sdk/workflow-tengo 2.2.1 → 2.3.1

package/dist/tengo/lib/exec.internal.lib.tengo

@@ -123,7 +123,7 @@ _binPackageCallOptions := func(runOptions, variables, descriptor, softPackage) {
 	envs := ll.toStrict({})
 	args := []
 
-	// Real command shoule be:
+	// Real command should be:
 	//   [<runEnv bin>] [<runEnv options>] [<entrypoint bin>] [<entrypoint options>] <cmd> <args>
 	if (maps.containsKey(descriptor, "runEnv")) {
 		runEnvInfo := _loadRunEnvironment(descriptor, runOptions.software.name, softPackage)
@@ -198,7 +198,7 @@ _remotePackageCallOptions := func(runOptions) {
 binaryRunOptions := func(runOptions) {
 	if (is_undefined(ll.fromStrict(runOptions).software)) {
 		//
-		// Simple local command run, no software pacakges involved (backward compatibility)
+		// Simple local command run, no software packages involved (backward compatibility)
 		//
 		return _directCallOptions(runOptions)
 	}

package/dist/tengo/lib/exec.lib.tengo

@@ -114,6 +114,9 @@ builder := func() {
 		 * @param arg: string - the argument to add.
 		 */
 		arg: func(arg) {
+			if is_undefined(arg) {
+				ll.panic("exec: arg can't be undefined")
+			}
 			args = append(args, arg)
 			return self
 		},

package/dist/tengo/lib/exec.package.lib.tengo

@@ -38,10 +38,10 @@ localDescriptorScheme := {
 }
 
 descriptorScheme := {
-	"isDev,omitempty": "bool",
-	"binary,omitempty": binaryDescriptorScheme,
-	"asset,omitempty": assetDescriptorScheme,
-	"local,omitempty": localDescriptorScheme
+	"isDev,?": "bool",
+	"binary,?": binaryDescriptorScheme,
+	"asset,?": assetDescriptorScheme,
+	"local,?": localDescriptorScheme
 }
 
 softwareInfoScheme := {

package/dist/tengo/lib/file.internal.lib.tengo

@@ -33,8 +33,8 @@ RTYPE_BLOB_DOWNLOAD := { Name: "BlobDownload", Version: "2" }
 createBlobImportExternal := func(settings) {
 	validation.assertJsonSchema(settings, {
 		"dataURL": "string",
-		"dataHeaders,omitempty": [["string"]],
-		"dataVersion,omitempty": "string"
+		"dataHeaders,?": [["string"]],
+		"dataVersion,?": "string"
 	})
 
 	importer := smart.structBuilder(RTYPE_BLOB_IMPORT_EXTERNAL, json.encode(settings)).lockAndBuild()
@@ -55,8 +55,8 @@ createBlobImportInternal := func(settings) {
 	validation.assertJsonSchema(settings, {
 		"path": "string",
 		"storageId": "string",
-		"maxSpeedBytes,omitempty": "number",
-		"dataVersion,omitempty": "string"
+		"maxSpeedBytes,?": "number",
+		"dataVersion,?": "string"
 	})
 
 	importer := smart.structBuilder(RTYPE_BLOB_IMPORT_INTERNAL, json.encode(settings)).lockAndBuild()

package/dist/tengo/lib/maps.lib.tengo

@@ -89,8 +89,49 @@ filterKeysValues := func(map, predicateFnK, predicateFnV) {
 	return filter(map, func(k, v) { return predicateFnK(k) && predicateFnV(v) })
 }
 
+/**
+ * Do shallow merge of arbitrary number of maps.
+ * Add {... key: undefined ...} to remove the "key" from the map collected from maps before.
+ */
+merge := func(...maps) {
+	res := {}
+	for map in maps {
+		for k, v in map {
+			if is_undefined(v) {
+				delete(res, k)
+			} else {
+				res[k] = v
+			}
+		}
+	}
+	return res
+}
+
+/**
+ * Do deep merge of arbitrary number of maps.
+ * Add {... key: undefined ...} to remove the "key" from the map collected from maps before.
+ */
+deepMerge := undefined
+deepMerge = func(...maps) {
+	res := {}
+	for map in maps {
+		for k, v in map {
+			if is_undefined(v) {
+				delete(res, k)
+			} else if is_map(res[k]) && is_map(v) {
+				res[k] = deepMerge(res[k], v)
+			} else {
+				res[k] = v
+			}
+		}
+	}
+	return res
+}
+
 export ll.toStrict({
 	map : map,
+	merge: merge,
+	deepMerge: deepMerge,
 	mapKeys : mapKeys,
 	mapValues : mapValues,
 	filter : filter,

package/dist/tengo/lib/pframes.spec.lib.tengo

@@ -0,0 +1,136 @@
+/**
+ * PFrames spec library:
+ *  - well known domains, annotations and column names.
+ *  - utility methods to manipulate column and axis spec
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+json := import("json")
+
+//
+// Common schemas
+//
+
+P_AXIS_SPEC_SCHEMA := {
+	// `__options__,closed`: ``, // commented out to keep the room for future schema modifications
+	`type`: `string,regex=Int|Long|Float|Double|String`,
+	`name`: `string`,
+	`domain,?`: { any: `string` },
+	`annotations,?`: { any: `string` },
+	`parentAxes,?`: `any` // [ `number` ]
+}
+
+P_COLUMN_SPEC_SCHEMA := {
+	// `__options__,closed`: ``, // commented out to keep the room for future schema modifications
+	`kind`: `string,regex=PColumn`,
+	`valueType`: `string`,
+	`name`: `string`,
+	`domain,?`: { any: `string` },
+	`annotations,?`: { any: `string` },
+	`parentAxes,?`: `any`, // [ `number` ]
+	`axesSpec,?`: [ P_AXIS_SPEC_SCHEMA ]
+}
+
+KIND_P_COLUMN := "PColumn"
+
+//
+// Column / axis names
+//
+
+/** Column containing labels for axis keys */
+N_LABEL := "pl7.app/label"
+
+//
+// Column / axis domains
+//
+
+/** Domain marking column or axis as block-specific, value must contain block id */
+D_BLOCK := "pl7.app/block"
+
+//
+// Column / axis annotations
+//
+
+/** Annotation with axis or column label */
+A_LABEL := "pl7.app/label"
+/**
+ * Annotation containing trace of operations resulting in a corresponding column.
+ *
+ * Value must containe JSON encoded array of structures:
+ *   {
+ *      type: string       - type of the step, often equals to block package name (i.e. milaboratories.mixcr-clonotyping)
+ *      label: string      - string representation of the operation often a block title
+ *      importance: number - higher the number higher the change label from trace step will be used to derive column label
+ *      id: string         - unique identifier of operation, often a block id
+ *   }
+ */
+A_TRACE := "pl7.app/trace"
+
+//
+// Utility methods
+//
+
+_TRACE_STEP_SCHEMA := {
+	`type`: `string`,
+	`label`: `string`,
+	`importance,?`: `number`,
+	`id,?`: `string`,
+	`__options__,closed`: ``
+}
+
+_TRACE_SCHEMA := [_TRACE_STEP_SCHEMA]
+
+/**
+ * Calculates content of pl7.app/trace annotation, given input column spec, or annotations section
+ * of specs, or directly content of pl7.app/trace from the input column spec.
+ *
+ * Basically validates and adds provided steps.
+ */
+makeTrace := func(input, ...steps) {
+	// validating input step structure
+	validation.assertJsonSchema(steps, _TRACE_SCHEMA)
+
+	currentTraceContent := undefined
+	if is_string(input) {
+		currentTraceContent = input
+	} else if is_string(input[A_TRACE]) {
+		currentTraceContent = input[A_TRACE]
+	} else if is_string(input["annotations"][A_TRACE]) {
+		currentTraceContent = input["annotations"][A_TRACE]
+	}
+
+	currentTrace := []
+	if !is_undefined(currentTraceContent) {
+		currentTrace = json.decode(currentTraceContent)
+		// validating trace we just parsed
+		validation.assertJsonSchema(currentTrace, _TRACE_SCHEMA)
+	}
+
+	currentTrace = append(currentTrace, steps...)
+	currentTraceStr := string(json.encode(currentTrace))
+
+	return ll.toStrict({
+		value: currentTrace,
+		valueStr: currentTraceStr,
+		/** Injects trace information into column spec */
+		inject: func(spec) {
+			validation.assertJsonSchema(spec, P_COLUMN_SPEC_SCHEMA)
+			return maps.deepMerge(spec, {annotations: {"pl7.app/trace": currentTraceStr}})
+		}
+	})
+}
+
+export ll.toStrict({
+	KIND_P_COLUMN: KIND_P_COLUMN,
+
+	N_LABEL: N_LABEL,
+
+	D_BLOCK: D_BLOCK,
+
+	A_LABEL: A_LABEL,
+	A_TRACE: A_TRACE,
+
+	makeTrace: makeTrace
+})

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -54,33 +54,33 @@ _SPEC_AXES_SCHEMA := {
 	// Regular expression, if matched - whole row would be skipped;
 	// Performed just after reading XSV before any other operations;
 	// default: no filter
-	`filterOutRegex,omitempty`: `string`,
+	`filterOutRegex,?`: `string`,
 	// Pre-processing operations to be performed just after filtering out rows
-	`preProcess,omitempty`: [_PRE_PROCESS_STEP_SCHEMA],
+	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
 	// Regular expression, if matched - value is considered to be N/A;
 	// default: Int, Long - check that the value is a valid integer number;
 	//          Float, Double - check that the value is a valid float number, not a NaN;
 	//          String - accept anything;
-	`naRegex,omitempty`: `string`,
+	`naRegex,?`: `string`,
 	// Should the N/A values (or empty rows) be allowed;
 	// default: false
-	`allowNA,omitempty`: `bool`,
+	`allowNA,?`: `bool`,
 	// Specification of the axis
 	`spec`: {
 		// Type of axis values
 		`type`: `string,regex=Int|Long|Float|Double|String`,
 		// Name of the axis to be used in spec;
 		// default: column label
-		`name,omitempty`: `string`,
+		`name,?`: `string`,
 		// Auxiliary information to the axis name, type and parents to form a unique identifier;
 		// default: empty
-		`domain,omitempty`: { any: `string` },
+		`domain,?`: { any: `string` },
 		// Any additional information attached to the axis that does not affect its identifier;
 		// default: "pl7.app/label": Column label from XSV
-		`annotations,omitempty`: { any: `string` },
+		`annotations,?`: { any: `string` },
 		// A list of zero-based indices of parent axes in the overall axes specification;
 		// default: empty
-		`parentAxes,omitempty`: [`number`]
+		`parentAxes,?`: [`number`]
 	}
 }
 
@@ -90,36 +90,36 @@ _SPEC_COLUMN_SCHEMA := {
 	// Regular expression, if matched - whole row would be skipped;
 	// Performed just after reading XSV before any other operations;
 	// default: no filter
-	`filterOutRegex,omitempty`: `string`,
+	`filterOutRegex,?`: `string`,
 	// Pre-processing operations to be performed just after filtering out rows
-	`preProcess,omitempty`: [_PRE_PROCESS_STEP_SCHEMA],
+	`preProcess,?`: [_PRE_PROCESS_STEP_SCHEMA],
 	// Regular expression, if matched - value is considered to be N/A;
 	// default: Int, Long - check that the value is a valid integer number;
 	//          Float, Double - check that the value is a valid float number, not a NaN;
 	//          String - accept anything;
-	`naRegex,omitempty`: `string`,
+	`naRegex,?`: `string`,
 	// Should the N/A values (or empty rows) be allowed;
 	// default: true
-	`allowNA,omitempty`: `bool`,
+	`allowNA,?`: `bool`,
 	// ID of the column to be used in spec and as a saved filename;
 	// default: column label with all special characters replaced with '_'
-	`id,omitempty`: `string`,
+	`id,?`: `string`,
 	// Specification of the column
 	`spec`: {
 		// Type of column values
 		`valueType`: `string,regex=Int|Long|Float|Double|String`,
 		// Name of the column to be used in spec;
 		// default: column label
-		`name,omitempty`: `string`,
+		`name,?`: `string`,
 		// Auxiliary information to the column name, type and parents to form a unique identifier;
 		// default: empty
-		`domain,omitempty`: { any: `string` },
+		`domain,?`: { any: `string` },
 		// Any additional information attached to the axis that does not affect its identifier;
 		// default: "pl7.app/label": Column label from XSV
-		`annotations,omitempty`: { any: `string` },
+		`annotations,?`: { any: `string` },
 		// A list of zero-based indices of parent axes in the overall axes specification;
 		// default: empty
-		`parentAxes,omitempty`: [ `number` ]
+		`parentAxes,?`: [ `number` ]
 	}
 }
 
@@ -128,86 +128,86 @@ _SPEC_INDEX_SCHEMA := {
 	`name`: `string`,
 	// Auxiliary information to the column name, type and parents to form a unique identifier;
 	// default: empty
-	`domain,omitempty`: { any: `string` },
+	`domain,?`: { any: `string` },
 	// Any additional information attached to the axis that does not affect its identifier;
 	// default: "pl7.app/label": name
-	`annotations,omitempty`: { any: `string` },
+	`annotations,?`: { any: `string` },
 	// A list of zero-based indices of parent axes in the overall axes specification;
 	// default: empty
-	`parentAxes,omitempty`: [`number`]
+	`parentAxes,?`: [`number`]
 }
 
 PFCONV_IMPORT_CFG_SCHEMA := {
 	// Single ASCII character to be used as separator;
 	// default: ',' for .csv, '\t' for .tsv
-	`separator,omitempty`: `string`,
+	`separator,?`: `string`,
 	// Single ASCII character, if XSV row begins with this character - the row would be skipped;
 	// default: error is thrown if comment lines are found in document
-	`commentLinePrefix,omitempty`: `string`,
+	`commentLinePrefix,?`: `string`,
 	// Should empty lines be skipped;
 	// default: false
-	`skipEmptyLines,omitempty`: `bool`,
+	`skipEmptyLines,?`: `bool`,
 	// Resolve duplicates by adding sequential suffixes to column labels;
 	// default: true
-	`allowColumnLabelDuplicates,omitempty`: `bool`,
+	`allowColumnLabelDuplicates,?`: `bool`,
 	// XSV columns to use as PColumn axes, order of axes in resulting columns
 	// would match the order of spec entries provided here
 	`axes`: [_SPEC_AXES_SCHEMA],
 	// Axis of type Long with XSV row numbers (would be the last one);
 	// default: do not create additional axis with indexes
-	`index,omitempty`: _SPEC_INDEX_SCHEMA,
+	`index,?`: _SPEC_INDEX_SCHEMA,
 	// XSV columns to use as PColumn values, each exported individually
 	`columns`: [_SPEC_COLUMN_SCHEMA],
 	// When columns spec is provided but no such column was found in XSV - create such column
 	// filled with NA values instead of failing;
 	// default: false
-	`allowArtificialColumns,omitempty`: `bool`,
+	`allowArtificialColumns,?`: `bool`,
 	// Prefix all column names with given string;
 	// default: names would be preserved
-	`columnNamePrefix,omitempty`: `string`,
+	`columnNamePrefix,?`: `string`,
 	// Columns would be stored using specified format;
 	// default: Binary
-	`storageFormat,omitempty` : `string,regex=Binary|Json`,
+	`storageFormat,?` : `string,regex=Binary|Json`,
 	// Partitioning key length;
 	// default: 0
-	`partitionKeyLength,omitempty` : `number`
+	`partitionKeyLength,?` : `number`
 }
 
 _COLUMN_FILTER_SCHEMA := {
 	// Match any of the types listed here;
 	// default: type is ignored during matching
-	`type,omitempty`: `string,regex=Int|Long|Float|Double|String|Bytes`,
+	`type,?`: `string,regex=Int|Long|Float|Double|String|Bytes`,
 	// Match any of the names listed here;
 	// default: name is ignored during matching
-	`name,omitempty`: `string`,
+	`name,?`: `string`,
 	// Match requires all the listed annotations to have provided values;
 	// default: no annotation equality checks are performed during matching
-	`annotationValue,omitempty`: { any: `string` },
+	`annotationValue,?`: { any: `string` },
 	// Match requires all the listed annotations to match provided regex patterns;
 	// default: no annotation pattern matching is performed during matching
-	`annotationPattern,omitempty`: { any: `string` }
+	`annotationPattern,?`: { any: `string` }
 }
 
 PFCONV_EXPORT_CFG_SCHEMA := {
 	// String representing NA values of types Int, Long, Float, Double;
 	// default: NaN
-	`naStr,omitempty`: `string`,
+	`naStr,?`: `string`,
 	// String representing NA and missing values of types String, Bytes;
 	// default: null
-	`nullStr,omitempty`: `string`,
+	`nullStr,?`: `string`,
 	// Toggle to export or skip axes and columns of type Bytes;
 	// default: false
-	`exportBytes,omitempty`: `bool`,
+	`exportBytes,?`: `bool`,
 	// Single ASCII character to be used as separator;
 	// default: ',' for .csv, '\t' for .tsv
-	`separator,omitempty`: `string`,
+	`separator,?`: `string`,
 	// Only columns matching the selector are exported;
 	// default: all columns are exported
-	`columnSelector,omitempty`: [_COLUMN_FILTER_SCHEMA],
+	`columnSelector,?`: [_COLUMN_FILTER_SCHEMA],
 	// During export, columns with equal axes specifications would be joined into single table
 	// using the join type specified;
 	// default: Full
-	`joinType,omitempty`: `string,regex=Full|Inner`
+	`joinType,?`: `string,regex=Full|Inner`
 }
 
 /**

package/dist/tengo/lib/tpl.lib.tengo

@@ -292,7 +292,6 @@ tpl := func() {
 				ll.assert(is_map(r), "expected map, got ", r)
 
 				for name, ref in r {
-					ll.assert(smart.isReference(ref), "expected reference, got %v", ref)
 					ll.assert(is_undefined(newInputs[name]), "prepare name is already in use: %s", name)
 
 					builder.add(name, ref)

package/dist/tengo/lib/validation.lib.tengo

@@ -234,22 +234,41 @@ _validateJson := func(js, schema, keyPath, ...stack) {
 	schemaTags := {}
 	// schema.key -> rule
 	schemaRules := {}
+	// js padded with all keys from schema
+	jsFull := {}
 
 	for k, v in schema {
 		p := _parseTags(k)
 		schemaTags[p.key] = p
 		schemaRules[p.key] = v
+		jsFull[p.key] = undefined
 	}
 
 	// tags that should be applied to all elements
 	any := schemaTags["any"]
+	if !is_undefined(any) {
+		if is_undefined(jsFull["any"]) { // in case actual object has field named "any"
+			delete(jsFull, "any")
+		}
+	}
 
 	options := schemaTags["__options__"]
 	if is_undefined(options) {
 		options = {}
+	} else {
+		delete(schemaTags, "__options__")
+		delete(schemaRules, "__options__")
+		if is_undefined(jsFull["__options__"]) { // in case actual object has field named "__options__"
+			delete(jsFull, "__options__")
+		}
 	}
 
-	for _key, elem in js {
+	// transferring fields from input map into padded map
+	for k, v in js {
+		jsFull[k] = v
+	}
+
+	for _key, elem in jsFull {
 		if any {
 			r := _validateJson(elem, schemaRules["any"], _formatKeyPath(keyPath, "%s.%s", _key), js, stack...)
 			if isFail(r) {
@@ -284,8 +303,11 @@ _validateJson := func(js, schema, keyPath, ...stack) {
 			}
 		}
 
-		if key.tags["omitempty"] && is_undefined(elem) {
-			continue
+		if is_undefined(elem) {
+			if key.tags["omitempty"] || key.tags["optional"] || key.tags["?"] {
+				continue
+			}
+			return fail(keyPath, "value %q does not contain key %q", js, _key)
 		}
 
 		r := _validateJson(elem, schemaRules[_key], _formatKeyPath(keyPath, "%s.%s", _key), js, stack...)
@@ -356,7 +378,7 @@ resource := func(...args) {
 	}
 
 	assertJsonSchema(ops, {
-		`type,omitempty`: {
+		`type,?`: {
 			Name: `string`,
 			Version: `string`
 		}

package/dist/tengo/lib/workflow.bobject.lib.tengo

@@ -15,20 +15,20 @@ _pObjectSpecSchema := {
 
 	"kind": "string",
 
-	"valueType,omitempty": `string,regex=json|File|DynamicFile|String|Double|Long|Int`,
+	"valueType,?": `string,regex=json|File|DynamicFile|String|Double|Long|Int`,
 
-	"name,omitempty": {",any": "string"},
+	"name,?": {",any": "string"},
 
-	"domain,omitempty": {",any": "string"},
+	"domain,?": {",any": "string"},
 
-	"axesSpec,omitempty": [{
+	"axesSpec,?": [{
 		"type": "string,regex=String",
 		"name": "string",
-		"domain,omitempty": {",any": "string"},
-		"annotations,omitempty": {",any": "string"}
+		"domain,?": {",any": "string"},
+		"annotations,?": {",any": "string"}
 	}],
 
-	"annotations,omitempty": {",any": "string"}
+	"annotations,?": {",any": "string"}
 }
 
 

package/dist/tengo/lib/workflow.bquery.lib.tengo

@@ -9,11 +9,11 @@ _CONTEXT_FIELD_NAME := "ctx"
 
 // options scheme for query
 _OPTIONS_SCHEME := {
-	"spec,omitempty": `bool`,
-	"data,omitempty": `bool`,
-	"single,omitempty": `bool`,
-	"first,omitempty": `bool`,
-	"errIfMissing,omitempty": `bool`
+	"spec,?": `bool`,
+	"data,?": `bool`,
+	"single,?": `bool`,
+	"first,?": `bool`,
+	"errIfMissing,?": `bool`
 }
 
 // default query options

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "2.2.1",
+  "version": "2.3.1",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",