@platforma-sdk/workflow-tengo 4.6.5 → 4.7.1

package/dist/tengo/lib/exec.constants.lib.tengo

@@ -1,21 +1,25 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
 ARG_TYPE_STRING := "string"
 ARG_TYPE_EXPRESSION := "expressionRef"
 ARG_TYPE_VAR        := "variableRef" // TODO: deprecated, remove after 15 Mar 2025
 ARG_TYPES := [ARG_TYPE_STRING, ARG_TYPE_EXPRESSION, ARG_TYPE_VAR]
 
+BATCH_QUEUE    := "batch" // special queue in storage controller with no limits
 HEAVY_QUEUE    := "heavy"
 MEDIUM_QUEUE   := "medium"
 LIGHT_QUEUE    := "light"
 UI_TASKS_QUEUE := "ui-tasks"
 
-export {
+export ll.toStrict({
 	ARG_TYPE_STRING: ARG_TYPE_STRING,
 	ARG_TYPE_EXPRESSION: ARG_TYPE_EXPRESSION,
 	ARG_TYPE_VAR: ARG_TYPE_VAR,
 	ARG_TYPES: ARG_TYPES,
 
+	BATCH_QUEUE: BATCH_QUEUE,
 	UI_TASKS_QUEUE: UI_TASKS_QUEUE,
 	LIGHT_QUEUE: LIGHT_QUEUE,
 	MEDIUM_QUEUE: MEDIUM_QUEUE,
 	HEAVY_QUEUE: HEAVY_QUEUE
-}
+})

package/dist/tengo/lib/exec.lib.tengo

@@ -759,12 +759,15 @@ builder := func() {
 			}
 
 			serviceExecInputs := {}
-			if feats.workdirLimits || feats.computeLimits {
-				serviceExecInputs.quota = limits.quotaBuilder().
-					queue(queue).
-					cpu(cpuRequest). // 'undefined' value is OK and makes runner to apply default queue limits
-					ram(ramRequest). // 'undefined' value is OK and makes runner to apply default queue limits
-					build()
+			if feats.workdirLimits || feats.computeLimits || limits.useBatch(queue) {
+				if (is_int(ramRequest)) {
+					ramRequest = string(ramRequest) + "b"
+				}
+				serviceExecInputs.quota = {
+					queue: queue,
+					cpu: cpuRequest,
+					ram: ramRequest
+				}
 			}
 
 

package/dist/tengo/lib/exec.limits.lib.tengo

@@ -9,8 +9,11 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 
 oop := import("@platforma-sdk/workflow-tengo:oop")
 validation := import("@platforma-sdk/workflow-tengo:validation")
+slices := import("@platforma-sdk/workflow-tengo:slices")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 json := import("json")
+constants := import("@platforma-sdk/workflow-tengo:exec.constants")
+feats := import("@platforma-sdk/workflow-tengo:feats")
 
 _RTYPE_QUOTA := { Name: "Quota", Version: "1" }
 
@@ -19,6 +22,16 @@ _RTYPE_QUOTA := { Name: "Quota", Version: "1" }
 
 
 
+
+useBatch := func(queueName) {
+	return feats.hasBatch && !slices.hasElement([ constants.UI_TASKS_QUEUE ], queueName)
+}
+
+
+
+
+
+
 quotaBuilder := func() {
 	self := undefined
 
@@ -132,9 +145,11 @@ storageSpaceRequestBuilder := func() {
 			requestBuilder := smart.ephemeralBuilder(_RTYPE_STORAGE_SPACE_REQUEST)
 			requestBuilder.getField(_SSR_FIELD_REQUEST).set(quota)
 
+			requestRef := requestBuilder.lockAndBuild()
+
 			return ll.toStrict(
-				oop.inherit(requestBuilder, {
-					allocation: requestBuilder.getField(_SSR_FIELD_ALLOCATION)
+				oop.inherit(requestRef, {
+					allocation: requestRef.getField(_SSR_FIELD_ALLOCATION)
 				})
 			)
 		}
@@ -143,8 +158,9 @@ storageSpaceRequestBuilder := func() {
 	return self
 }
 
-_RTYPE_COMPUTE_REQUEST              := func(executor) { return { Name: "ComputeRequest/" + executor, Version: "1" } }
-_CR_DEFAULT_EXECUTOR_ID             := "executor"
+_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1  := { Name: "ComputeRequest/executor", Version: "1" }
+_RTYPE_COMPUTE_REQUEST_BATCH_V1     := { Name: "ComputeRequest/batch", Version: "1" }
+
 _CR_FIELD_REQUEST                   := "request"    // input
 _CR_FIELD_ALLOCATION                := "allocation" // output
 
@@ -152,7 +168,7 @@ computeRequestBuilder := func() {
 	self := undefined
 
 	quota := undefined
-	executor := _CR_DEFAULT_EXECUTOR_ID
+	queue_name := undefined
 
 	self = ll.toStrict({
 
@@ -173,10 +189,9 @@ computeRequestBuilder := func() {
 
 
 
-
-		executor: func(execID) {
-			validation.assertType(execID, "string", "exec.limits.executor: executor ID should be a string")
-			executor = execID
+		queue: func(name) {
+			validation.assertType(name, "string", "exec.limits.queue: queue name should be a string")
+			queue_name = name
 
 			return self
 		},
@@ -188,14 +203,23 @@ computeRequestBuilder := func() {
 
 		build: func() {
 			ll.assert(smart.isReference(quota), "exec.limits.build: quota of ComputeRequest must be a valid reference")
+			ll.assert(queue_name != "", "exec.limits.build: queue name is required for computeRequestBuilder")
+
+			requestBuilder := undefined
+
+			if useBatch(queue_name) {
+				requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_BATCH_V1)
+			} else {
+				requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1)
+			}
 
-			rType := _RTYPE_COMPUTE_REQUEST(executor)
-			requestBuilder := smart.ephemeralBuilder(rType)
 			requestBuilder.getField(_CR_FIELD_REQUEST).set(quota)
 
+			requestRef := requestBuilder.lockAndBuild()
+
 			return ll.toStrict(
-				oop.inherit(requestBuilder, {
-					allocation: requestBuilder.getField(_CR_FIELD_ALLOCATION)
+				oop.inherit(requestRef, {
+					allocation: requestRef.getField(_CR_FIELD_ALLOCATION)
 				})
 			)
 		}
@@ -205,6 +229,7 @@ computeRequestBuilder := func() {
 }
 
 export ll.toStrict({
+	useBatch:                        useBatch,
 	quotaBuilder:                    quotaBuilder,
 	storageSpaceRequestBuilder:      storageSpaceRequestBuilder,
 	computeRequestBuilder:           computeRequestBuilder

package/dist/tengo/lib/exec.runcmd.lib.tengo

@@ -15,7 +15,8 @@ slices := import("@platforma-sdk/workflow-tengo:slices")
 limits := import("@platforma-sdk/workflow-tengo:exec.limits")
 
 _RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
-_RTYPE_RUN_COMMAND_V2 := { Name: "RunCommand/executor", Version: "2" }
+_RTYPE_RUN_COMMAND_EXEC_V2 := { Name: "RunCommand/executor", Version: "2" }
+_RTYPE_RUN_COMMAND_BATCH_V2 := { Name: "RunCommand/batch", Version: "2" }
 _RTYPE_RUN_COMMAND_CMD := { Name: "RunCommandCmd", Version: "1" }
 _RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
 _RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
@@ -77,16 +78,6 @@ _createExpressionArg := func(argTpl) {
 
 
 builder := func(workdir) {
-	res := undefined
-
-	if feats.computeLimits {
-		res = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_V2)
-	} else {
-		res = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_V1)
-	}
-
-	res.getField(_FIELD_WORKDIR_IN).set(workdir)
-
 	self := undefined
 
 	cmd := undefined
@@ -105,7 +96,7 @@ builder := func(workdir) {
 	stderr := "stderr.txt"
 	nErrorLines := 200
 
-	self = ll.toStrict(oop.inherit(res, {
+	self = ll.toStrict({
 
 
 
@@ -387,6 +378,25 @@ builder := func(workdir) {
 
 
 		build: func() {
+			runRes := undefined
+
+			useBatch := limits.useBatch(queue)
+
+			if useBatch {
+
+
+
+				runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_BATCH_V2)
+
+			} else if feats.computeLimits {
+				runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_EXEC_V2)
+
+			} else {
+				runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_V1)
+			}
+
+			runRes.getField(_FIELD_WORKDIR_IN).set(workdir)
+
 
 
 			ll.assert(!hasPatternArgs || len(substRules) != 0,
@@ -396,21 +406,23 @@ builder := func(workdir) {
 				"runcmd.builder().build: expression arguments must be enabled only if platforma backend supports expressions, found: %v",
 				hasExpressionArgs)
 
-			if feats.computeLimits {
+			if feats.computeLimits || useBatch {
 				if !smart.isReference(_allocation) {
 
 
-					_quota := limits.quotaBuilder().queue(queue).build()
-					_allocation = limits.computeRequestBuilder().quota(_quota).build().allocation
+					_quotaRef := limits.quotaBuilder().queue(queue).build()
+					_resReqRef := limits.computeRequestBuilder().queue(queue).quota(_quotaRef).build()
+					_allocation = _resReqRef.allocation
 				}
-				self.getField(_FIELD_ALLOCATION).set(_allocation)
+
+				runRes.getField(_FIELD_ALLOCATION).set(_allocation)
 			}
 
 			refsRes := smart.createMapResourceWithType(_RTYPE_RUN_COMMAND_REFS, refs)
-			self.getField(_FIELD_REFS).set(refsRes)
+			runRes.getField(_FIELD_REFS).set(refsRes)
 
 			cmdRes := smart.createValueResource(_RTYPE_RUN_COMMAND_CMD, json.encode(cmd))
-			self.getField(_FIELD_CMD).set(cmdRes)
+			runRes.getField(_FIELD_CMD).set(cmdRes)
 
 			options := {
 				queueName: queue,
@@ -423,19 +435,19 @@ builder := func(workdir) {
 			}
 
 			optionsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_OPTIONS, json.encode(options))
-			self.getField(_FIELD_OPTIONS).set(optionsRes)
+			runRes.getField(_FIELD_OPTIONS).set(optionsRes)
 
 			argsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_ARGS, json.encode(args))
-			self.getField(_FIELD_ARGS).set(argsRes)
+			runRes.getField(_FIELD_ARGS).set(argsRes)
 
-			res := ll.toStrict(oop.inherit(self.lockAndBuild(), {
-				"workdir": self.outputs()[_FIELD_WORKDIR_OUT],
+			res := ll.toStrict(oop.inherit(runRes.lockAndBuild(), {
+				"workdir": runRes.outputs()[_FIELD_WORKDIR_OUT],
 				"options": options
 			}))
 
 			return res
 		}
-	}))
+	})
 
 	return self
 }

package/dist/tengo/lib/feats.lib.tengo

@@ -18,9 +18,25 @@ _isEnabled := func(featName) {
 	return _checker(featName)
 }
 
+_executors := undefined
+_hasRunner := func(e_id) {
+	if is_undefined(_executors) {
+		_executors = {}
+		for e in plapi.getExecutors() {
+			_executors[e] = true
+		}
+	}
+
+	return !is_undefined(_executors[e_id])
+}
+
 export ll.toStrict({
+	hasExec:                    _hasRunner("executor"),     // local binary execution is available
+	hasBatch:                   _hasRunner("batch"),    // batch system execution is available
+
 	pureFutureFields:           _isEnabled("pureFutureFields")   || _api >= 1, // 'pure' future fields magic that provides deduplication
 	commandExpressions:         _isEnabled("commandExpressions") || _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )
 	workdirLimits:              _isEnabled("workdirLimits")      || _api >= 4, // workdir storage controller can limit concurrency
-	computeLimits:              _isEnabled("computeLimits")                    // runner can plan CPU and RAM
+	computeLimits:              _isEnabled("computeLimits"),                   // runner can plan CPU and RAM
+	resourceDuplicateEvent:     false                                          // template can subscribe to ResourceDuplicate event
 })

package/dist/tengo/lib/smart.lib.tengo

@@ -685,6 +685,22 @@ resource = func(r) {
 
 
 
+		createDynamicField: func(name) {
+			if is_bytes(name) {
+				name = string(name)
+			}
+			ll.assert(is_string(name) && name != "", "field name must be a non-empty string")
+			fid := ll.fieldId(self.id, name)
+			tx.createField(fid, constants.FTYPE_DYNAMIC)
+			return field(fid)
+		},
+
+
+
+
+
+
+
 
 		kvGet: func(key) {
 			return tx.resourceKeyValueGet(self.id, key)

package/dist/tengo/lib/tpl.base.lib.tengo

@@ -102,7 +102,8 @@ AWAIT_TEMPLATES := {
 	"PColumnBundleWithPartitions": [
 		[{ wildcard: "*" }, { match: "^ref$" }, "ResourceReady"], // optional ref field
 		[{ wildcard: "*" }, "spec", "ResourceReady"],
-		[{ wildcard: "*" }, "data", "InputsLocked"]
+		[{ wildcard: "*" }, "data", "InputsLocked"],
+		[{ wildcard: "*" }, "data", { match: "^\\{.*\\}$" }, "InputsLocked"] // for multi-hit BQuery results; key is always ref
 	]
 }
 
@@ -132,6 +133,9 @@ tpl := func(ops) {
 	_stateRequests := []
 	_delegatedOutputs := []
 
+
+
+
 	self = {
 
 
@@ -565,7 +569,7 @@ tpl := func(ops) {
 			if _hasPrepared {
 				_stateRequests = append(_stateRequests, {
 					path: [tplutil.PREPARE_MAP_FIELD_NAME],
-					state: "ResourceReady"
+					state: "Final"
 				})
 			}
 

package/dist/tengo/lib/tpl.state-await.lib.tengo

@@ -7,6 +7,9 @@ tx := import("tx")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 text := import("text")
 regexp := import("@platforma-sdk/workflow-tengo:regexp")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+feats := import("@platforma-sdk/workflow-tengo:feats")
 
 createContext := func() {
 	return {
@@ -15,6 +18,9 @@ createContext := func() {
 	}
 }
 
+
+RTYPE_AWAITER := { Name: "awaiter", Version: "1" }
+
 awaitOneState := func(field, path, state) {
 	if field.isErrorSet() && !field.isValueSet() {
 
@@ -51,9 +57,6 @@ awaitOneState := func(field, path, state) {
 					subscriptions: [{
 						resource: resource.id,
 						event: "FieldCreated"
-					},{
-						resource: resource.id,
-						event: "InputsLocked"
 					}]
 				}
 			}
@@ -93,19 +96,19 @@ awaitOneState := func(field, path, state) {
 			}
 
 			subscriptions := []
-			for nextFieldName, nextField in resource.inputs() {
+			maps.forEach(resource.inputs(), func(nextFieldName, nextField) {
 				if !matcher(nextFieldName) {
-					continue
+					return
 				}
 				res := awaitOneState(nextField, rPath, state)
 				if res.type == "Ready" {
-					continue
+					return
 				} else if res.type == "Subscribe" {
 					subscriptions = append(subscriptions, res.subscriptions...)
 				} else {
 					ll.panic("Unknown result type: %v", res.type)
 				}
-			}
+			})
 			if len(subscriptions) == 0 {
 				return { type: "Ready" }
 			} else {
@@ -157,7 +160,7 @@ awaitOneState := func(field, path, state) {
 						}]
 					}
 				}
-			} else if state == "ResourceReady" {
+			} else if state == "ResourceReady" || state == "Final" {
 
 
 				if resource.isFinal() {
@@ -167,7 +170,7 @@ awaitOneState := func(field, path, state) {
 						type: "Subscribe",
 						subscriptions: [{
 							resource: resource.id,
-							event: "ResourceReady"
+							event: "Final" // will also effectively subscribe to ResourceDuplicate and ResourceError
 						}]
 					}
 				}
@@ -178,19 +181,21 @@ awaitOneState := func(field, path, state) {
 	}
 }
 
+_renderer := smart.resource(ll.getCurrentTemplateRenderer())
+
 
 await := func(rootField, ctxStorageResource, requests) {
 	pssKey := "await:" + string(rootField.id.ResourceID) + ":" + rootField.id.Name
 	prevSubscriptions := ctxStorageResource.kvGetAsJson(pssKey)
 
-	if !is_undefined(prevSubscriptions) && len(prevSubscriptions) == 0 {
+	lastSubscriptionKey := undefined
+	if !is_undefined(prevSubscriptions) {
+		if is_undefined(prevSubscriptions.lastSubscriptionKey) {
 
-		ll.panic("Unexpected state, desired state was reached in the previous call")
-	}
-
-	if is_undefined(prevSubscriptions) {
-
-		prevSubscriptions = {}
+			ll.panic("Unexpected state, desired state was reached in the previous call")
+		} else {
+			lastSubscriptionKey = prevSubscriptions.lastSubscriptionKey
+		}
 	}
 
 	subscriptions := {}
@@ -222,34 +227,88 @@ await := func(rootField, ctxStorageResource, requests) {
 	}
 
 
-	for key, sub in subscriptions {
-		if !is_undefined(prevSubscriptions[key]) {
 
-			continue
+
+
+
+
+
+
+	firstSubscriptionKey := undefined
+	firstSubscription := undefined
+	lastSubscriptionKeyPresent := false
+	maps.forEach(subscriptions, func(key, sub) {
+		if is_undefined(firstSubscriptionKey) {
+			firstSubscriptionKey = key
+			firstSubscription = sub
 		}
-		e := {}
-		e[sub.event] = true
-		tx.subscribeTo(sub.resource, pssKey + ":" + key, e)
+		if !is_undefined(lastSubscriptionKey) && lastSubscriptionKey == key {
+			lastSubscriptionKeyPresent = true
+		}
+	})
+
+
+
+	if lastSubscriptionKeyPresent {
+		return false
 	}
 
 
-	for key, _ in prevSubscriptions {
-		if !is_undefined(subscriptions[key]) {
+	if !is_undefined(firstSubscriptionKey) {
 
-			continue
+		fullEventId := pssKey + ":" + firstSubscriptionKey
+		targetResource := firstSubscription.resource
+		targetEvents := [firstSubscription.event]
+
+		if firstSubscription.event == "Final" { // special case
+			if !feats.resourceDuplicateEvent {
+
+
+
+
+
+
+				target := smart.resource(targetResource)
+				isEphemeral := target.info().Features.ephemeral
+				if !isEphemeral { // doing proxy-object workaround for pure resources only
+					awaiterBuilder := smart.ephemeralBuilder(RTYPE_AWAITER)
+					awaiterBuilder.createInputField("target").set(target)
+					awaiter := awaiterBuilder.lockAndBuild()
+					_renderer.createDynamicField("tmp:" + fullEventId).set(awaiter)
+					targetResource = awaiter.info().ID // converting to global resource id; this is a hack to overcome the fact that tx.subscribeTo can't work with local ids
+				}
+				targetEvents = ["ResourceReady", "ResourceError"]
+
+			} else {
+				targetEvents = ["ResourceReady", "ResourceError", "ResourceDuplicate"]
+			}
+		} else if firstSubscription.event == "FieldCreated" { // another special case
+
+			targetEvents = ["FieldCreated", "InputsLocked"]
 		}
-		tx.unsubscribeFrom(pssKey + ":" + key)
+
+		e := {}
+		for event in targetEvents {
+			e[event] = true
+		}
+		tx.subscribeTo(targetResource, fullEventId, e)
+
+	}
+
+
+	if !is_undefined(lastSubscriptionKey) {
+		tx.unsubscribeFrom(pssKey + ":" + lastSubscriptionKey)
 	}
 
 
 	newPrevSubscriptions := {}
-	for key, _ in subscriptions {
-		newPrevSubscriptions[key] = true
+	if !is_undefined(firstSubscriptionKey) {
+		newPrevSubscriptions.lastSubscriptionKey = firstSubscriptionKey
 	}
 
 	ctxStorageResource.kvSetJson(pssKey, newPrevSubscriptions)
 
-	return len(subscriptions) == 0
+	return is_undefined(firstSubscriptionKey)
 }
 
 export ll.toStrict({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "4.6.5",
+  "version": "4.7.1",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",