@platforma-sdk/workflow-tengo 4.8.1 → 4.9.0

package/dist/tengo/lib/exec.runcmd.lib.tengo

@@ -14,6 +14,8 @@ constants := import("@platforma-sdk/workflow-tengo:exec.constants")
 slices := import("@platforma-sdk/workflow-tengo:slices")
 limits := import("@platforma-sdk/workflow-tengo:exec.limits")
 
+delayedCompAlloc := import("@platforma-sdk/workflow-tengo:exec.delayed-compute-allocation")
+
 _RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
 _RTYPE_RUN_COMMAND_EXEC_V2 := { Name: "RunCommand/executor", Version: "2" }
 _RTYPE_RUN_COMMAND_BATCH_V2 := { Name: "RunCommand/batch", Version: "2" }
@@ -22,6 +24,42 @@ _RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
 _RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
 _RTYPE_RUN_COMMAND_REFS := { Name: "RunCommandRefs", Version: "1" }
 
+ARG_SCHEMA := {
+	"__options__,closed": true,
+	"type": "string,regex=string|expressionRef|variableRef",
+	"value": "string"
+}
+
+RUN_CMD_PLAN_REFS_SCHEMA := "any"
+
+RUN_CMD_PLAN_OPTIONS_SCHEMA := {
+	"cmd,?": ARG_SCHEMA,
+	"args": [ARG_SCHEMA],
+	"envs": { "any": ARG_SCHEMA },
+	"substRules": { "any": "string" },
+	"customPaths": [ARG_SCHEMA]
+}
+
+RUN_CMD_PLAN_SCHEMA := {
+	"refs": RUN_CMD_PLAN_REFS_SCHEMA,
+	"options": RUN_CMD_PLAN_OPTIONS_SCHEMA
+}
+
+_initStateFromPlan := func(state, initState, context) {
+	validation.assertType(initState, RUN_CMD_PLAN_SCHEMA, "invalid initial state for " + context)
+
+	initOptions := initState.options
+	state.options.cmd = maps.clone(initOptions.cmd)
+	state.options.args = maps.clone(initOptions.args)
+	state.options.envs = maps.clone(initOptions.envs)
+	state.options.substRules = maps.clone(initOptions.substRules)
+	state.options.customPaths = maps.clone(initOptions.customPaths)
+
+	for k, v in initState.refs {
+		state.refs[k] = v
+	}
+}
+
 
 _FIELD_ALLOCATION := "allocation" // not in v1
 _FIELD_REFS := "refs"
@@ -38,7 +76,7 @@ _FIELD_WORKDIR_OUT := "workdirOut"
 
 
 
-_createSimpleArg := func(argument) {
+simpleArg := func(argument) {
 	return {
 		type: constants.ARG_TYPE_STRING,
 		value: argument
@@ -50,7 +88,7 @@ _createSimpleArg := func(argument) {
 
 
 
-_createVariableArg := func(argTpl) {
+variableArg := func(argTpl) {
 	return {
 		type: constants.ARG_TYPE_VAR,
 		value: argTpl
@@ -64,58 +102,29 @@ _createVariableArg := func(argTpl) {
 
 
 
-_createExpressionArg := func(argTpl) {
+expressionArg := func(argTpl) {
 	return {
 		type: constants.ARG_TYPE_EXPRESSION,
 		value: argTpl
 	}
 }
 
-
-
-
-
-
-
-builder := func(workdir) {
+_createAbstractRunCommandPlan := func(state, returnSelfFn) {
 	self := undefined
 
-	cmd := undefined
-	args := []
-	envs := {}
-	queue := ""
-	refs := {}
-	_allocation := undefined
-
-	substRules := {}
-	hasPatternArgs := false // TODO: deprecated, remove after a month of expressionRefs in prod.
-	hasExpressionArgs := false
-	customPaths := []
-
-	stdout := "stdout.txt"
-	stderr := "stderr.txt"
-	nErrorLines := 200
-
 	self = ll.toStrict({
 
 
 
 
 
-		allocation: func(allocation) {
-			validation.assertType(allocation, validation.reference, "runcmd.builder: allocation should be a valid reference")
-			_allocation = allocation
-			return self
-		},
-
-
-
-
-
-
 		cmd: func(commandName) {
-			cmd = _createSimpleArg(commandName)
-			return self
+			if ll.isMap(commandName) {
+				self.cmdTyped(commandName.type, commandName.value)
+			} else {
+				state.options.cmd = simpleArg(commandName)
+			}
+			return returnSelfFn()
 		},
 
 
@@ -127,9 +136,8 @@ builder := func(workdir) {
 
 
 		cmdVar: func(commandTpl) {
-			cmd = _createVariableArg(commandTpl)
-			hasPatternArgs = true
-			return self
+			state.options.cmd = variableArg(commandTpl)
+			return returnSelfFn()
 		},
 
 
@@ -141,14 +149,13 @@ builder := func(workdir) {
 
 
 		cmdExpression: func(commandTpl) {
-			cmd = _createExpressionArg(commandTpl)
-			hasExpressionArgs = true
-			return self
+			state.options.cmd = expressionArg(commandTpl)
+			return returnSelfFn()
 		},
 
 		cmdTyped: func(type, value) {
 			ll.assert(slices.hasElement(constants.ARG_TYPES, type),
-				"runcmd.builder().cmdTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+				"runcmd.plan.cmdTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 
 			if type == constants.ARG_TYPE_STRING {
 				return self.cmd(value)
@@ -158,7 +165,7 @@ builder := func(workdir) {
 				return self.cmdVar(value)
 			}
 
-			ll.panic("runcmd.builder().cmdTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+			ll.panic("runcmd.plan.cmdTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 		},
 
 
@@ -167,8 +174,12 @@ builder := func(workdir) {
 
 
 		arg: func(arg) {
-			args = append(args, _createSimpleArg(arg))
-			return self
+			if ll.isMap(arg) {
+				self.argTyped(arg.type, arg.value)
+			} else {
+				state.options.args = append(state.options.args, simpleArg(arg))
+			}
+			return returnSelfFn()
 		},
 
 
@@ -180,9 +191,8 @@ builder := func(workdir) {
 
 
 		argVar: func(argTpl) {
-			args = append(args, _createVariableArg(argTpl))
-			hasPatternArgs = true
-			return self
+			state.options.args = append(state.options.args, variableArg(argTpl))
+			return returnSelfFn()
 		},
 
 
@@ -194,14 +204,13 @@ builder := func(workdir) {
 
 
 		argExpression: func(argTpl) {
-			args = append(args, _createExpressionArg(argTpl))
-			hasExpressionArgs = true
-			return self
+			state.options.args = append(state.options.args, expressionArg(argTpl))
+			return returnSelfFn()
 		},
 
 		argTyped: func(type, value) {
 			ll.assert(slices.hasElement(constants.ARG_TYPES, type),
-				"runcmd.builder().argTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+				"runcmd.plan.argTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 
 			if type == constants.ARG_TYPE_STRING {
 				return self.arg(value)
@@ -211,7 +220,15 @@ builder := func(workdir) {
 				return self.argVar(value)
 			}
 
-			ll.panic("runcmd.builder().argTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+			ll.panic("runcmd.plan.argTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+		},
+
+
+
+
+		resetArgs: func() {
+			state.options.args = []
+			return returnSelfFn()
 		},
 
 
@@ -222,13 +239,27 @@ builder := func(workdir) {
 
 		ref: func(refKey, ref) {
 			ll.assert(smart.isReference(ref),
-				"runcmd.builder().ref() must be a reference (valid field or resource) %s", refKey)
+				"runcmd.plan.ref() must be a reference (valid field or resource) %s", refKey)
 
-			ll.assert(!maps.containsKey(refs, refKey),
+			ll.assert(!maps.containsKey(state.refs, refKey),
 				"attempt to override existing reference %q", refKey)
 
-			refs[refKey] = ref
-			return self
+			state.refs[refKey] = ref
+			return returnSelfFn()
+		},
+
+
+
+
+
+
+
+
+
+		refVar: func(varName, refKey, ref) {
+			self.ref(refKey, ref)
+			self.substitutionRule(varName, refKey)
+			return returnSelfFn()
 		},
 
 
@@ -238,8 +269,12 @@ builder := func(workdir) {
 
 
 		env: func(name, value) {
-			envs[name] = value
-			return self
+			if ll.isMap(value) {
+				self.envTyped(name, value.type, value.value)
+			} else {
+				state.options.envs[name] = simpleArg(value)
+			}
+			return returnSelfFn()
 		},
 
 
@@ -250,9 +285,8 @@ builder := func(workdir) {
 
 
 		envVar: func(name, valueTpl) {
-			envs[name] = _createVariableArg(valueTpl)
-			hasPatternArgs = true
-			return self
+			state.options.envs[name] = variableArg(valueTpl)
+			return returnSelfFn()
 		},
 
 
@@ -263,14 +297,13 @@ builder := func(workdir) {
 
 
 		envExpression: func(name, valueTpl) {
-			envs[name] = _createExpressionArg(valueTpl)
-			hasExpressionArgs = true
-			return self
+			state.options.envs[name] = expressionArg(valueTpl)
+			return returnSelfFn()
 		},
 
 		envTyped: func(name, type, value) {
 			ll.assert(slices.hasElement(constants.ARG_TYPES, type),
-				"runcmd.builder().envTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+				"runcmd.plan.envTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 
 			if type == constants.ARG_TYPE_STRING {
 				return self.env(name, value)
@@ -280,7 +313,7 @@ builder := func(workdir) {
 				return self.envVar(name, value)
 			}
 
-			ll.panic("runcmd.builder().envTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+			ll.panic("runcmd.plan.envTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 		},
 
 
@@ -288,24 +321,50 @@ builder := func(workdir) {
 
 
 
+		addPath: func(path) {
+			if ll.isMap(path) {
+				self.addPathTyped(path.type, path.value)
+			} else {
+				state.options.customPaths = append(state.options.customPaths, simpleArg(path))
+			}
+			return returnSelfFn()
+		},
+
+
 
-		addToPath: func(type, value) {
+
+
+
+
+		addPathVar: func(pathTpl) {
+			state.options.customPaths = append(state.options.customPaths, variableArg(pathTpl))
+			return returnSelfFn()
+		},
+
+
+
+
+
+
+
+		addPathExpression: func(pathTpl) {
+			state.options.customPaths = append(state.options.customPaths, expressionArg(pathTpl))
+			return returnSelfFn()
+		},
+
+		addPathTyped: func(type, value) {
 			ll.assert(slices.hasElement(constants.ARG_TYPES, type),
-				"runcmd.builder().addToPath: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+				"runcmd.plan.addPathTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 
 			if type == constants.ARG_TYPE_STRING {
-				customPaths = append(customPaths, _createSimpleArg(value))
+				return self.addPath(value)
 			} else if type == constants.ARG_TYPE_EXPRESSION {
-				customPaths = append(customPaths, _createExpressionArg(value))
-				hasExpressionArgs = true
+				return self.addPathExpression(value)
 			} else if type == constants.ARG_TYPE_VAR {
-				customPaths = append(customPaths, _createVariableArg(value))
-				hasPatternArgs = true
-			} else {
-				ll.panic("runcmd.builder().addToPath: unknown type %s, expect: %v", type, constants.ARG_TYPES)
+				return self.addPathVar(value)
 			}
 
-			return self
+			ll.panic("runcmd.plan.addPathTyped: unknown type %s, expect: %v", type, constants.ARG_TYPES)
 		},
 
 
@@ -316,8 +375,8 @@ builder := func(workdir) {
 
 
 		substitutionRule: func(varName, refKey) {
-			substRules[varName] = refKey
-			return self
+			state.options.substRules[varName] = refKey
+			return returnSelfFn()
 		},
 
 
@@ -327,8 +386,116 @@ builder := func(workdir) {
 
 		substitutionRules: func(rules) {
 			for k, v in rules {
-				substRules[k] = v
+				state.options.substRules[k] = v
+			}
+			return returnSelfFn()
+		}
+	})
+
+	return self
+}
+
+createRunCommandPlan := func(...initialState) {
+	state := {
+		refs: {},
+		options: {
+			cmd: undefined,
+			args: [],
+			envs: {},
+			substRules: {},
+			customPaths: []
+		}
+	}
+
+	if len(initialState) == 1 {
+		initState := initialState[0]
+		_initStateFromPlan(state, initState, "run command plan")
+	} else {
+		ll.assert(len(initialState) == 0, "createRunCommandPlan accepts at most one argument")
+	}
+
+	self := undefined
+
+	self = oop.inherit(_createAbstractRunCommandPlan(state, func() { return self }), {
+		state: func() {
+			newState := {
+				options: maps.clone(state.options),
+				refs: {}
 			}
+			for k, v in state.refs {
+				newState.refs[k] = v
+			}
+			return newState
+		}
+	})
+
+	return self
+}
+
+
+
+
+
+
+
+builder := func(workdir, ...initialState) {
+	self := undefined
+
+	state := {
+		refs: {},
+		options: {
+			cmd: undefined,
+			args: [],
+			envs: {},
+			substRules: {},
+			customPaths: []
+		}
+	}
+
+	if len(initialState) == 1 {
+		initState := initialState[0]
+		_initStateFromPlan(state, initState, "run command builder")
+	} else {
+		ll.assert(len(initialState) == 0, "builder accepts at most two arguments")
+	}
+
+	allocation := undefined
+	queue := undefined
+	cpu := undefined
+	ram := undefined
+
+	stdout := "stdout.txt"
+	stderr := "stderr.txt"
+	nErrorLines := 200
+
+	self = oop.inherit(_createAbstractRunCommandPlan(state, func() { return self }), {
+
+
+
+
+
+		allocation: func(value) {
+			validation.assertType(value, validation.reference, "runcmd.builder: allocation should be a valid reference")
+			allocation = value
+			return self
+		},
+
+
+
+
+
+
+		request: func(request) {
+			validation.assertType(request, {
+				"queue": "string",
+				"cpu,?": "number",
+				"ram,?": ["or", "number", "string"]
+			}, "runcmd.builder.request: request must be a valid request")
+
+			queue = request.queue
+			cpu = request.cpu
+			ram = request.ram
+
 			return self
 		},
 
@@ -347,6 +514,29 @@ builder := func(workdir) {
 
 
 
+		mem: func(value) {
+			validation.assertType(value, ["or", "number", "string"], "exec.builder.mem: RAM amount should be a number or string")
+			ll.assert(is_string(value) || value > 0, "exec.builder.mem: amount in bytes should be greater than 0")
+			ram = value
+			return self
+		},
+
+
+
+
+
+
+		cpu: func(value) {
+			validation.assertType(value, "number", "runcmd.builder.cpu: value must be a number")
+			cpu = value
+			return self
+		},
+
+
+
+
+
+
 		stdout: func(fileName) {
 			stdout = fileName
 			return self
@@ -379,6 +569,45 @@ builder := func(workdir) {
 
 		build: func() {
 			runRes := undefined
+			options := state.options
+			refs := state.refs
+
+			hasPatternArgs := false
+			hasExpressionArgs := false
+
+			check := func(arg) {
+				if !ll.isMap(arg) {
+					return
+				}
+				if arg.type == constants.ARG_TYPE_VAR {
+					hasPatternArgs = true
+				}
+				if arg.type == constants.ARG_TYPE_EXPRESSION {
+					hasExpressionArgs = true
+				}
+			}
+
+			check(options.cmd)
+
+			for arg in options.args {
+				check(arg)
+			}
+
+			for _, env in options.envs {
+				check(env)
+			}
+
+			for path in options.customPaths {
+				check(path)
+			}
+
+
+
+
+			ll.assert(is_undefined(allocation) || (is_undefined(cpu) && is_undefined(ram)),
+				"runcmd.builder.build: allocation cannot be set along with cpu or ram")
+
+			ll.assert(!is_undefined(queue), "queue must be set")
 
 			useBatch := limits.useBatch(queue)
 
@@ -399,51 +628,60 @@ builder := func(workdir) {
 
 
 
-			ll.assert(!hasPatternArgs || len(substRules) != 0,
+			ll.assert(!hasPatternArgs || len(options.substRules) != 0,
 				"found pattern argument, but no substitution rules were set: hasPatternArgs: %v, substRules: %v, cmd: %v, args: %v, envs: %v",
-				hasPatternArgs, substRules, cmd, args, envs)
+				hasPatternArgs, options.substRules, options.cmd, options.args, options.envs)
 			ll.assert(!hasExpressionArgs || feats.commandExpressions,
 				"runcmd.builder().build: expression arguments must be enabled only if platforma backend supports expressions, found: %v",
 				hasExpressionArgs)
 
-			if feats.computeLimits || useBatch {
-				if !smart.isReference(_allocation) {
+			if !is_undefined(allocation) {
+				runRes.getField(_FIELD_ALLOCATION).set(allocation)
+			} else {
+				request := {
+					queue: queue
+				}
+				if !is_undefined(cpu) {
+					request.cpu = cpu
+				}
+				if !is_undefined(ram) {
+					request.ram = ram
+				}
+
+
+
 
 
-					_quotaRef := limits.quotaBuilder().queue(queue).build()
-					_resReqRef := limits.computeRequestBuilder().queue(queue).quota(_quotaRef).build()
-					_allocation = _resReqRef.allocation
-				}
 
-				runRes.getField(_FIELD_ALLOCATION).set(_allocation)
+				runRes.getField(_FIELD_ALLOCATION).set(delayedCompAlloc.create(workdir, request))
 			}
 
 			refsRes := smart.createMapResourceWithType(_RTYPE_RUN_COMMAND_REFS, refs)
 			runRes.getField(_FIELD_REFS).set(refsRes)
 
-			cmdRes := smart.createValueResource(_RTYPE_RUN_COMMAND_CMD, json.encode(cmd))
+			cmdRes := smart.createValueResource(_RTYPE_RUN_COMMAND_CMD, json.encode(options.cmd))
 			runRes.getField(_FIELD_CMD).set(cmdRes)
 
-			options := {
+			buildOptions := {
 				queueName: queue,
 				errorLines: nErrorLines,
 				redirectStdout: stdout,
 				redirectStderr: stderr,
-				env: envs,
-				additionalPaths: customPaths,
-				substitutions: substRules
+				env: options.envs,
+				additionalPaths: options.customPaths,
+				substitutions: options.substRules
 			}
 
-			optionsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_OPTIONS, json.encode(options))
+			optionsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_OPTIONS, json.encode(buildOptions))
 			runRes.getField(_FIELD_OPTIONS).set(optionsRes)
 
-			argsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_ARGS, json.encode(args))
+			argsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_ARGS, json.encode(options.args))
 			runRes.getField(_FIELD_ARGS).set(argsRes)
 
-			res := ll.toStrict(oop.inherit(runRes.lockAndBuild(), {
+			res := oop.inherit(runRes.lockAndBuild(), {
 				"workdir": runRes.outputs()[_FIELD_WORKDIR_OUT],
-				"options": options
-			}))
+				"options": buildOptions
+			})
 
 			return res
 		}
@@ -453,5 +691,13 @@ builder := func(workdir) {
 }
 
 export ll.toStrict({
-	builder: builder
+	builder: builder,
+	simpleArg: simpleArg,
+	variableArg: variableArg,
+	expressionArg: expressionArg,
+	createRunCommandPlan: createRunCommandPlan,
+	ARG_SCHEMA: ARG_SCHEMA,
+	RUN_CMD_PLAN_REFS_SCHEMA: RUN_CMD_PLAN_REFS_SCHEMA,
+	RUN_CMD_PLAN_OPTIONS_SCHEMA: RUN_CMD_PLAN_OPTIONS_SCHEMA,
+	RUN_CMD_PLAN_SCHEMA: RUN_CMD_PLAN_SCHEMA
 })

package/dist/tengo/lib/feats.lib.tengo

@@ -32,7 +32,7 @@ _hasRunner := func(e_id) {
 
 export ll.toStrict({
 	hasExec:                    _hasRunner("executor"),     // local binary execution is available
-	hasBatch:                   _hasRunner("batch"),    // batch system execution is available
+	hasBatch:                   _hasRunner("batch"),        // batch system execution is available
 
 	pureFutureFields:           _isEnabled("pureFutureFields")   || _api >= 1, // 'pure' future fields magic that provides deduplication
 	commandExpressions:         _isEnabled("commandExpressions") || _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )

package/dist/tengo/lib/feature-guard.lib.tengo

@@ -0,0 +1,14 @@
+
+
+
+
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+feat := import("@platforma-sdk/workflow-tengo:feats")
+
+if(!feat.commandExpressions) {
+	ll.panic("INCOMPATIBLE PLATFORMA BACKEND, PLEASE UPGRADE PLATFORMA TO THE LATEST VERSION! " +
+		"(commandExpressions feature is not supported)")
+}
+
+export {}