@platforma-sdk/workflow-tengo 2.11.0 → 2.12.1

package/dist/tengo/lib/constants.lib.tengo

@@ -28,6 +28,7 @@ RTYPE_QUERY_RESULT        := { Name: "BQueryResult",      Version: "1" }
 RTYPE_OBJECT_SPEC         := { Name: "BObjectSpec",       Version: "1" }
 
 RTYPE_WORKDIR_CREATE      := { Name: "WorkdirCreate",     Version: "1" }
+RTYPE_WORKDIR_CREATE_2    := { Name: "WorkdirCreate",     Version: "2" }
 RTYPE_WORKDIR_FILL        := { Name: "WorkdirFill",       Version: "1" }
 RTYPE_WORKDIR_FILL_RULES  := { Name: "WorkdirFill/rules", Version: "1" }
 RTYPE_WORKDIR_SAVE        := { Name: "WorkdirSave",       Version: "1" }
@@ -67,6 +68,7 @@ export ll.toStrict({
 	RTYPE_RENDER_EPH_TEMPLATE   : RTYPE_RENDER_EPH_TEMPLATE,
 	RTYPE_RENDER_TEMPLATE       : RTYPE_RENDER_TEMPLATE,
 	RTYPE_WORKDIR_CREATE        : RTYPE_WORKDIR_CREATE,
+	RTYPE_WORKDIR_CREATE_2      : RTYPE_WORKDIR_CREATE_2,
 	RTYPE_WORKDIR_FILL          : RTYPE_WORKDIR_FILL,
 	RTYPE_WORKDIR_FILL_RULES    : RTYPE_WORKDIR_FILL_RULES,
 	RTYPE_WORKDIR_SAVE          : RTYPE_WORKDIR_SAVE,

package/dist/tengo/lib/exec.lib.tengo

@@ -17,6 +17,7 @@ enum := import("enum")
 oop := import("@platforma-sdk/workflow-tengo:oop")
 validation := import("@platforma-sdk/workflow-tengo:validation")
 constants := import("@platforma-sdk/workflow-tengo:constants")
+limits := import("@platforma-sdk/workflow-tengo:exec.limits")
 monetization := import("@platforma-sdk/workflow-tengo:exec.monetization")
 execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
 
@@ -50,7 +51,14 @@ builder := func() {
 			"either a command with arguments or a software were already set.")
 	}
 
+	//
+	// Limits
+	//
 	queue := _HEAVY_QUEUE
+	hasLimits := false
+	cpuRequest := undefined
+	ramRequest := undefined
+
 	cache := 0
 	nErrorLines := 200
 
@@ -262,6 +270,40 @@ builder := func() {
 			return self
 		},
 
+		/**
+		 * Preserve CPUs in environments that plan CPU resources (like PBS or Slurm).
+		 *
+		 * @param amount: number - number of cores requested for command.
+		 */
+		cpu: func(amount) {
+			validation.assertType(amount, "number", "exec.builder.cpu: amount of cores must be a number")
+			ll.assert(amount > 0, "exec.builder.cpu: amount of cores should be greater than 0")
+			hasLimits = true
+			cpuRequest = amount
+
+			return self
+		},
+
+		/**
+		 * Amount of RAM to request.
+		 *
+		 * @param amount: number | string - amount of RAM in bytes or string with size suffix (case-insensitive):
+		 *                                     K,  KB,  M,  MB,  G,  GB for base-10 sizes (x1000)
+		 *                                    Ki, KiB, Mi, MiB, Gi, GiB for base-2 sizes (x1024)
+		 *                                  when operating with bytes, you may use 'units' package for convenience:
+		 *                                    120 * units.GiB
+		 *
+		 * @return builder
+		 */
+		ram: func(amount) {
+			validation.assertType(amount, ["or", "number", "string"], "exec.builder.ram: RAM amount should be a number or string")
+			ll.assert(is_string(amount) || amount > 0, "exec.builder.ram: amount in bytes should be greater than 0")
+			hasLimits = true
+			ramRequest = amount
+
+			return self
+		},
+
 		/**
 		 * Use cacheMillis, cacheSeconds, cacheMinutes, cacheHours and cacheDays to not make hidden mistakes in cache time ranges.
 		 * Sets the cache time.
@@ -684,7 +726,7 @@ builder := func() {
 				nErrorLines: nErrorLines
 			}
 
-			execInputs := {
+			pureExecInputs := {
 				filesToAdd: smart.createMapResource(filesToAdd),
 				filesToWrite: smart.createMapResource(filesToWriteRefs),
 				dirsToCreate: slices.fromSet(dirsToCreate),
@@ -701,30 +743,39 @@ builder := func() {
 				assetCache: assetCache
 			}
 
+			serviceExecInputs := {}
+			if hasLimits && feats.workdirLimits {
+				serviceExecInputs.quota = limits.quotaBuilder().
+					queue(queue).
+					cpu(cpuRequest).
+					ram(ramRequest).
+					build()
+			}
+
 			// We add monetization only if it's enabled
 			// to not break a deduplication of exec's
 			// that were created before this feature existed.
 			if monetization.shouldRun(mnz) {
-				execInputs.runOptions.monetization = monetization.toJson(mnz)
+				pureExecInputs.runOptions.monetization = monetization.toJson(mnz)
 			}
 
 			// We add new argsTyped and envsTyped only when they were added
 			// to not break a deduplication on old pl backend.
 			if typedArgSet || typedEnvSet {
-				execInputs.runOptions.argsTyped = argsTyped
-				execInputs.runOptions.envsTyped = envsTyped
+				pureExecInputs.runOptions.argsTyped = argsTyped
+				pureExecInputs.runOptions.envsTyped = envsTyped
 			}
 
 			if !is_undefined(soft) {
-				execInputs.softwareInfo = soft.resource
-				execInputs.runOptions.software = {
+				pureExecInputs.softwareInfo = soft.resource
+				pureExecInputs.runOptions.software = {
 					name: soft.name,
 					version: soft.version,
 					descriptor: ll.fromStrict(soft.descriptor)
 				}
 			}
 
-			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.exec"), execInputs)
+			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.exec"), pureExecInputs, serviceExecInputs)
 
 			self := undefined
 			self = ll.toStrict(oop.inherit(tpl, {

package/dist/tengo/lib/exec.limits.lib.tengo

@@ -0,0 +1,149 @@
+/**
+ * A library that creates Quota resource.
+ * Shared with 'workdir' package.
+ *
+ * Was shaped as a separate lib to avoid recursive imports (exec -> workdir -> exec)
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+
+smart := import("@platforma-sdk/workflow-tengo:smart")
+json := import("json")
+
+_RTYPE_QUOTA := { Name: "Quota", Version: "1" }
+
+/**
+ * Builder for quota resource.
+ *
+ * Creates 'Quota' resource used to request specific amount of CPU and RAM in a given queue
+ */
+quotaBuilder := func() {
+	self := undefined
+
+	queue_name := undefined
+	cpu_request := undefined
+	ram_request := undefined
+
+
+	self = ll.toStrict({
+		/**
+		 * Create quota resource for specific execution queue.
+		 * This is the only method required for call before .build()
+		 *
+		 * @param name: string - name of the queue.
+		 * @return builder
+		 */
+		queue: func(name) {
+			validation.assertType(name, "string", "exec.quota.builder.queue: name of the queue should be a string")
+			queue_name = name
+
+			return self
+		},
+
+		/**
+		 * Amount of cores to request in quota.
+		 *
+		 * @param amount: number - number of cores.
+		 * @return builder
+		 */
+		cpu: func(amount) {
+			validation.assertType(amount, ["or", "number", "null"], "exec.quota.builder.cpu: CPU amount should be a number")
+			ll.assert(is_undefined(amount) || amount > 0, "exec.quota.builder.cpu: amount of cores should be greater than 0")
+			cpu_request = amount
+
+			return self
+		},
+
+		/**
+		 * Amount of RAM to request.
+		 *
+		 * @param amount: number | string - amount of RAM in bytes or string with size suffix (case-insensitive):
+		 *                                    K,  KB,  M,  MB,  G,  GB for base-10 sizes (x1000)
+		 *                                   Ki, KiB, Mi, MiB, Gi, GiB for base-2 sizes (x1024)
+		 *
+		 * @return builder
+		 */
+		ram: func(amount) {
+			validation.assertType(amount, ["or", "number", "string", "null"], "exec.quota.builder.ram: RAM amount should be a number or string")
+			ll.assert(is_undefined(amount) || is_string(amount) || amount > 0, "exec.quota.builder.ram: amount in bytes should be greater than 0")
+			ram_request = amount
+
+			return self
+		},
+
+		/**
+		 * Create Quota resource.
+		 *
+		 * @return smart.resource - reference to the resulting value resource
+		 */
+		build: func() {
+			ll.assert(queue_name != "", "exec.quota.builder: queue name is mandatory and could not be empty string")
+
+			data := {
+				queue: queue_name
+			}
+			if !is_undefined(cpu_request) {
+				data.cpuCores = cpu_request
+			}
+			if !is_undefined(ram_request) {
+				data.memory = ram_request
+			}
+
+			return smart.ephemeralBuilder(_RTYPE_QUOTA, json.encode(data)).lockAndBuild()
+		}
+	})
+
+	return self
+}
+
+_RTYPE_STORAGE_SPACE_REQUEST    := { Name: "StorageSpaceRequest", Version: "1" }
+_SSR_FIELD_REQUEST                  := "request"    // input
+_SSR_FIELD_ALLOCATION               := "allocation" // output
+
+storageSpaceRequestBuilder := func() {
+	self := undefined
+
+	quota := undefined
+
+	self = ll.toStrict({
+		/**
+		 * Request for given resources
+		 *
+		 * @param ref: smart.reference - reference to Quota resource to be fulfilled by storage allocator
+		 * @return spaceRequestBuilder
+		 */
+		quota: func(ref) {
+			validation.assertType(ref, validation.reference, "workdir.limits.quota: quota must be a valid reference to Quota resource")
+			quota = ref
+
+			return self
+		},
+
+		/**
+		 * Build StorageSpaceRequest resource
+		 *
+		 * @return StorageSpaceRequest
+		 */
+		build: func() {
+			ll.assert(smart.isReference(quota), "workdir.limits.build: quota of StorageSpaceRequest must be a valid reference")
+
+			requestBuilder := smart.ephemeralBuilder(_RTYPE_STORAGE_SPACE_REQUEST)
+			requestBuilder.getField(_SSR_FIELD_REQUEST).set(quota)
+
+			return ll.toStrict(
+				oop.inherit(requestBuilder, {
+					allocation: requestBuilder.getField(_SSR_FIELD_ALLOCATION)
+				})
+			)
+		}
+	})
+
+	return self
+}
+
+export ll.toStrict({
+	quotaBuilder:                    quotaBuilder,
+	storageSpaceRequestBuilder:      storageSpaceRequestBuilder
+})

package/dist/tengo/lib/exec.runcmd.lib.tengo

@@ -18,6 +18,9 @@ _RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
 _RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
 _RTYPE_RUN_COMMAND_REFS := { Name: "RunCommandRefs", Version: "1" }
 
+_FIELD_WORKDIR_IN := "workdirIn"
+_FIELD_WORKDIR_OUT := "workdirOut"
+
 /**
  * Creates a simple argument for the command.
  *
@@ -64,7 +67,7 @@ _createExpressionArg := func(argTpl) {
  */
 builder := func(workdir) {
 	res := smart.ephemeralBuilder(_RTYPE_RUN_COMMAND)
-	res.getField("workdirIn").set(workdir)
+	res.getField(_FIELD_WORKDIR_IN).set(workdir)
 
 	self := undefined
 
@@ -386,7 +389,7 @@ builder := func(workdir) {
 			self.getField("args").set(argsRes)
 
 			res := ll.toStrict(oop.inherit(self.lockAndBuild(), {
-				"workdir": self.outputs().workdirOut,
+				"workdir": self.outputs()[_FIELD_WORKDIR_OUT],
 				"options": options
 			}))
 

package/dist/tengo/lib/feats.lib.tengo

@@ -11,5 +11,6 @@ if is_undefined(_api) {
 
 export ll.toStrict({
     pureFutureFields:           _api >= 1,
-    commandExpressions:         _api >= 2
+    commandExpressions:         _api >= 2,
+    workdirLimits:              _api >= 4
 })

package/dist/tengo/lib/render.lib.tengo

@@ -20,9 +20,14 @@ _SUBSCRIPTION_FIELD := "subscription"
  * @param tpl: template resource (see assets.importTemplate())
  * @param isEph: whether template is ephemeral
  * @param inputs: a map of inputs to be passed to the template
+ * @param serviceInputs: (optional) a map of service inputs to be passed to the template.
+ *                                  Service inputs are not accounted in dedupliation and recovering.
+ *                                  Two resources with the same <inputs> and different <serviceInputs>
+ *                                  would be considered as copies of each-other.
  * @return renderer: a smart resource with additional output("name") method
  */
-createUniversal := func(tpl, isEph, inputs) {
+createUniversal := func(tpl, isEph, inputs, ...opts) {
+	serviceInputs := opts[0]
 
 	// build renderer resource
 	rbuilder := undefined
@@ -45,6 +50,9 @@ createUniversal := func(tpl, isEph, inputs) {
 	for name, value in inputs {
 		inputsMap.add(name, value)
 	}
+	for name, value in serviceInputs {
+		inputsMap.addService(name, value)
+	}
 
 	rbuilder.createInputField(tplutil.INPUTS_MAP_FIELD).set(inputsMap.build())
 	// build
@@ -76,10 +84,14 @@ createUniversal := func(tpl, isEph, inputs) {
  *
  * @param tpl: template resource (see assets.importTemplate())
  * @param inputs: a map of inputs to be passed to the template
+ * @param serviceInputs: (optional) a map of service inputs to be passed to the template.
+ *                                  Service inputs are not accounted in dedupliation and recovering.
+ *                                  Two resources with the same <inputs> and different <serviceInputs>
+ *                                  would be considered as copies of each-other.
  * @return renderer: a smart resource with additional output("name") method
  */
-create := func(tpl, inputs) {
-	return createUniversal(tpl, false, inputs)
+create := func(tpl, inputs, ...opts) {
+	return createUniversal(tpl, false, inputs, opts...)
 }
 
 /**
@@ -87,10 +99,14 @@ create := func(tpl, inputs) {
  *
  * @param tpl: template resource (see assets.importTemplate())
  * @param inputs: a map of inputs to be passed to the template
+ * @param serviceInputs: (optional) a map of service inputs to be passed to the template.
+ *                                  The split into <inputs> and <serviceInputs> has no
+ *                                  effect on ephemeral templates, as they do not
+ *                                  have deduplication.
  * @return renderer: a smart resource with additional output("name") method
  */
-createEphemeral := func(tpl, inputs) {
-	return createUniversal(tpl, true, inputs)
+createEphemeral := func(tpl, inputs, ...opts) {
+	return createUniversal(tpl, true, inputs, opts...)
 }
 
 export ll.toStrict({

package/dist/tengo/lib/smart.lib.tengo

@@ -929,7 +929,7 @@ resourceBuilder = func(r) {
 		 * Create new field with given name and field type.
 		 *
 		 * @param name field name
-		 * @param fieldType: field type - input, output, dynamic, service, OTW. Use FfTYPE_* constants.
+		 * @param fieldType: field type - input, output, dynamic, service, OTW. Use FTYPE_* constants.
 		 */
 		createField: func(name, fieldType) {
 			assertNotBuilt()
@@ -938,6 +938,17 @@ resourceBuilder = func(r) {
 			return field(fid)
 		},
 
+		/**
+		 * Create service field with given name.
+		 * Service fields behave very like inputs, but with deduplication disabled.
+		 * This means, that if you create 2 resources, each having 1 input field with value "A" and the service
+		 * field with different values, they would be considered the same by backend.
+		 */
+		createServiceField: func(name) {
+			assertNotBuilt()
+			return self.createField(name, constants.FTYPE_SERVICE)
+		},
+
 		/**
 		 * Create new input field with given name.
 		 *
@@ -945,7 +956,7 @@ resourceBuilder = func(r) {
 		 */
 		createInputField: func(name) {
 			assertNotBuilt()
-			return self.createField(name, "input")
+			return self.createField(name, constants.FTYPE_INPUT)
 		},
 
 		/**
@@ -955,7 +966,7 @@ resourceBuilder = func(r) {
 		 */
 		createOutputField: func(name) {
 			assertNotBuilt()
-			return self.createField(name, "output")
+			return self.createField(name, constants.FTYPE_OUTPUT)
 		},
 
 		/**
@@ -1086,6 +1097,18 @@ _mapBuilder := func(isEph) {
 			return self
 		},
 
+		/**
+		 * Adds object automatically converting non-reference objects into Json values.
+		 * The object is added into service field, meaning the value would not affect deduplication of the map.
+		 *
+		 * @param key: string - how the field will be named in the map
+		 * @param value: any - a reference or a primitive value
+		 * @return self: mapBuilder
+		 */
+		addService: func(key, value) {
+			builder.createServiceField(key).setRefOrJson(value)
+			return self
+		},
 
 		/**
 		 * Creates a map in the platforma

package/dist/tengo/lib/units.lib.tengo

@@ -0,0 +1,13 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+export ll.toStrict({
+	// Base-2 size modifiers:
+	KiB: 1024,
+	MiB: 1024 * 1024,
+	GiB: 1024 * 1024 * 1024,
+
+	// Decimal modifiers:
+	KB: 1000,
+	MB: 1000000,
+	GB: 1000000000
+})

package/dist/tengo/lib/validation.lib.tengo

@@ -399,7 +399,7 @@ resource := func(...args) {
 		ops = args[0]
 	}
 
-	assertJsonSchema(ops, {
+	assertType(ops, {
 		`type,?`: {
 			Name: `string`,
 			Version: `string`

package/dist/tengo/lib/workdir.lib.tengo

@@ -5,6 +5,8 @@
 json := import("json")
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
+feats := import("@platforma-sdk/workflow-tengo:feats")
+
 assets := import("@platforma-sdk/workflow-tengo:assets")
 constants := import("@platforma-sdk/workflow-tengo:constants")
 smart := import("@platforma-sdk/workflow-tengo:smart")
@@ -16,9 +18,36 @@ enum := import("enum")
 validation := import("@platforma-sdk/workflow-tengo:validation")
 render := import("@platforma-sdk/workflow-tengo:render")
 
+_WD_CREATE_FIELD_ALLOCATION := "allocation"
+_WD_CREATE_FIELD_WORKDIR    := "workdir"
+
+_WD_FILL_FIELD_WORKDIR_IN   := "workdirIn"
+_WD_FILL_FIELD_WORKDIR_OUT  := "workdirOut"
+
+/**
+ * Create empty working directory
+ *
+ * @return smart.field - field where working directory will appear after creation
+ */
 create := func() {
-	wd := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_CREATE).lockAndBuild()
-	return wd.getField("workdir")
+	// working directories without quotas (old-style)
+	wdCreate := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_CREATE).lockAndBuild()
+	return wdCreate.getField(_WD_CREATE_FIELD_WORKDIR)
+}
+
+/**
+ * Create empry working directory for given allocation. This means, the WD would actually be created only
+ * when allocation is ready for usage.
+ *
+ * @param allocation: smart.reference - a reference to storage space allocation.
+ *                                      It is usually obtained from StorageSpaceRequest
+ *                                      (see exec.limits.storageSpaceRequestBuilder)
+ */
+createV2 := func(allocation) {
+	wdCreate := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_CREATE_2).lockAndBuild()
+	wdCreate.getField(_WD_CREATE_FIELD_ALLOCATION).set(allocation)
+
+	return wdCreate.getField(_WD_CREATE_FIELD_WORKDIR)
 }
 
 _getFileFillRule := func(filePath, extension) {
@@ -87,14 +116,12 @@ _getDirFillRule := func(dir) {
 /**
  * Fills a workdir with files, values, and directories.
  *
- * @param workdir: reference - a reference to the workdir to fill.
  * @return self: builder - a builder that adds files and directories to the workdir. */
-_fill := func(workdir) {
-	ll.assert(smart.isReference(workdir), "workdir must be a reference to a resource or field")
-
+_fill := func() {
 	wd := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_FILL)
-	wd.getField("workdirIn").set(workdir)
 
+	/** @type reference - space allocation to be used for WD */
+	_allocation := undefined
 	blobs := {}
 	files := {}
 	zipExtractions := {}
@@ -113,6 +140,16 @@ _fill := func(workdir) {
 			sets.add(dirs, newDirs...)
 		},
 
+		/**
+		 * Use given storage allocation for working directory.
+		 *
+		 * @param allocation: reference
+		 */
+		allocation: func(allocation) {
+			validation.assertType(allocation, validation.reference, "workdir.builder: allocation should be a valid reference")
+			_allocation = allocation
+		},
+
 		/**
 		 * Adds files from given zip archive into working directory.
 		 *
@@ -223,6 +260,15 @@ _fill := func(workdir) {
 		 * @return wd: reference - a reference to the filled workdir.
 		 */
 		build: func() {
+			emptyWorkdir := undefined
+			if feats.workdirLimits && smart.isReference(_allocation) {
+				emptyWorkdir = createV2(_allocation)
+			} else {
+				emptyWorkdir = create()
+			}
+
+			wd.getField(_WD_FILL_FIELD_WORKDIR_IN).set(emptyWorkdir)
+
 			rules := []
 			for dir, _ in dirs {
 				rules = append(rules, _getDirFillRule(dir))
@@ -249,7 +295,7 @@ _fill := func(workdir) {
 			self.getField("dataIn").set(smart.createBinaryMapResource(values))
 			self.getField("blobsIn").set(smart.createBlobMapResource(blobs))
 
-			return self.lockAndBuild().outputs().workdirOut
+			return self.lockAndBuild().outputs()[_WD_FILL_FIELD_WORKDIR_OUT]
 		}
 	}))
 
@@ -262,7 +308,7 @@ _fill := func(workdir) {
  * @return self: builder - a builder that adds files and directories to the workdir.
  */
 builder := func() {
-	return _fill(create())
+	return _fill()
 }
 
 /**
@@ -368,5 +414,6 @@ save := func(workdir) {
 export ll.toStrict({
 	builder: builder,
 	create: create,
+	createV2: createV2,
 	save: save
 })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "2.11.0",
+  "version": "2.12.1",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -16,11 +16,11 @@
     "./dist/**/*"
   ],
   "dependencies": {
-    "@milaboratories/software-pframes-conv": "2.1.9",
+    "@milaboratories/software-pframes-conv": "2.1.12",
     "@platforma-open/milaboratories.software-small-binaries": "^1.15.6"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "^1.17.5"
+    "@platforma-sdk/tengo-builder": "^1.17.7"
   },
   "scripts": {
     "type-check": "pl-tengo check",