@platforma-sdk/workflow-tengo 6.8.3 → 6.10.0

package/dist/tengo/lib/exec.constants.lib.tengo

@@ -28,6 +28,7 @@ RUNNER_EXECUTOR := "executor" // local binary execution
 META_INPUT_MEM_FORMULA := "__memFormula__"
 META_INPUT_CPU_FORMULA := "__cpuFormula__"
 META_INPUT_GPU_FORMULA := "__gpuFormula__"
+META_INPUT_SCRATCH_FORMULA := "__scratchFormula__"
 REGULAR_INPUT_LINE_COUNT := "__metric_lineCount__"
 
 export ll.toStrict({
@@ -49,5 +50,6 @@ export ll.toStrict({
 	META_INPUT_MEM_FORMULA: META_INPUT_MEM_FORMULA,
 	META_INPUT_CPU_FORMULA: META_INPUT_CPU_FORMULA,
 	META_INPUT_GPU_FORMULA: META_INPUT_GPU_FORMULA,
+	META_INPUT_SCRATCH_FORMULA: META_INPUT_SCRATCH_FORMULA,
 	REGULAR_INPUT_LINE_COUNT: REGULAR_INPUT_LINE_COUNT
 })

package/dist/tengo/lib/exec.delayed-compute-allocation.lib.tengo

@@ -22,7 +22,8 @@ create := func(workdir, req) {
 		"queue": "string",
 		"cpu,?": "number",
 		"ram,?": ["or", "number", "string"],
-		"gpuMemory,?": "string"
+		"gpuMemory,?": "string",
+		"scratchFreeSpace,?": ["or", "number", "string"]
 	}, "request must be a valid request")
 
 	reqMap := { queue: req.queue }
@@ -35,6 +36,9 @@ create := func(workdir, req) {
 	if !is_undefined(req.gpuMemory) {
 		reqMap.gpuMemory = req.gpuMemory
 	}
+	if !is_undefined(req.scratchFreeSpace) {
+		reqMap.scratchFreeSpace = req.scratchFreeSpace
+	}
 
 	delayedComputeAllocation := render.createEphemeral(tpl, {
 		request: smart.createJsonResource(reqMap),

package/dist/tengo/lib/exec.formula.lib.tengo

@@ -298,12 +298,21 @@ referencedMetricFiles := func(resolvedNode) {
 
 
 
+
+
+
+
+
+
+
+
 
 
 
 applyFormulaFallback := func(dims) {
 	gpu := is_undefined(dims.gpu) ? {} : dims.gpu
-	res := { ram: dims.ram.static, cpu: dims.cpu.static, gpu: gpu.static, err: undefined }
+	scratch := is_undefined(dims.scratch) ? {} : dims.scratch
+	res := { ram: dims.ram.static, cpu: dims.cpu.static, gpu: gpu.static, scratch: scratch.static, err: undefined }
 	if !is_undefined(_unwrap(dims.ram.ast)) && is_undefined(res.ram) {
 		if is_undefined(dims.ram.fallback) {
 			res.err = "exec.builder: the RAM formula needs a static fallback on backends that can't evaluate resource formulas at run time. Provide .staticFallback(...) on the RAM formula, set a static ram, or upgrade the backend."
@@ -325,6 +334,9 @@ applyFormulaFallback := func(dims) {
 		}
 		res.gpu = gpu.fallback
 	}
+	if !is_undefined(_unwrap(scratch.ast)) && is_undefined(res.scratch) {
+		res.scratch = scratch.fallback
+	}
 	return res
 }
 

package/dist/tengo/lib/exec.lib.tengo

@@ -19,6 +19,7 @@ constants := import("@platforma-sdk/workflow-tengo:constants")
 canonical := import("@platforma-sdk/workflow-tengo:canonical")
 monetization := import("@platforma-sdk/workflow-tengo:exec.monetization_internal")
 execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
+tmpdirCompat := import("@platforma-sdk/workflow-tengo:exec.tmpdir-compat")
 execFormula := import("@platforma-sdk/workflow-tengo:exec.formula")
 units := import("@platforma-sdk/workflow-tengo:units")
 feats := import("@platforma-sdk/workflow-tengo:feats")
@@ -65,6 +66,16 @@ _assertValidGpuRequest := func(amount, ctx) {
 
 
 
+_assertValidScratchRequest := func(amount, ctx) {
+	validation.assertType(amount, ["or", "number", "string"], ctx + ": scratch space amount must be a number of bytes or a size string like '500GiB'")
+	ll.assert(is_string(amount) || amount == 0 || amount >= 16 * units.MiB, ctx + ": scratch space in bytes should be at least 16MiB, or exactly 0 to ask only for a temporary directory (did you pass bytes instead of GiB?)")
+}
+
+
+
+
+
+
 builder := func() {
 	self := undefined
 
@@ -92,6 +103,7 @@ builder := func() {
 	cpuRequest := undefined
 	ramRequest := undefined
 	gpuMemory := undefined
+	scratchFreeSpace := undefined
 
 
 
@@ -103,6 +115,8 @@ builder := func() {
 	cpuFormulaFallback := undefined
 	gpuFormulaAst := undefined
 	gpuFormulaFallback := undefined
+	scratchFormulaAst := undefined
+	scratchFormulaFallback := undefined
 	fileTags := {}   // fileName -> tag (for f.size("tag") / f.lineCount("tag") aggregation)
 
 	commandName := ""
@@ -520,6 +534,27 @@ builder := func() {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -592,6 +627,20 @@ builder := func() {
 					memFormulaFallback = undefined
 				}
 			}
+			setScratch := func(v, ctx) {
+				if execFormula.isFormula(v) {
+					scratchFormulaAst = execFormula.astOf(v)
+					scratchFormulaFallback = undefined
+					fb := execFormula.fallbackOf(v)
+					if !is_undefined(fb) { _assertValidScratchRequest(fb, ctx + " scratchFreeSpace fallback"); scratchFormulaFallback = fb }
+					scratchFreeSpace = undefined
+				} else {
+					_assertValidScratchRequest(v, ctx + " scratchFreeSpace")
+					scratchFreeSpace = v
+					scratchFormulaAst = undefined
+					scratchFormulaFallback = undefined
+				}
+			}
 			setVram := func(v, ctx) {
 				if execFormula.isFormula(v) {
 					gpuFormulaAst = execFormula.astOf(v)
@@ -610,8 +659,8 @@ builder := func() {
 			assertKeys := func(block, name, allowVram) {
 				ll.assert(ll.isMap(block), "exec.builder.resources.%s: expected a map", name)
 				for k, _ in block {
-					ok := k == "cpu" || k == "ram" || (allowVram && k == "vram")
-					ll.assert(ok, "exec.builder.resources.%s: unknown key %q (allowed: %s)", name, k, allowVram ? "cpu, ram, vram" : "cpu, ram")
+					ok := k == "cpu" || k == "ram" || k == "scratchFreeSpace" || (allowVram && k == "vram")
+					ll.assert(ok, "exec.builder.resources.%s: unknown key %q (allowed: %s)", name, k, allowVram ? "cpu, ram, vram, scratchFreeSpace" : "cpu, ram, scratchFreeSpace")
 				}
 				ll.assert(!is_undefined(block.cpu) && !is_undefined(block.ram),
 					"exec.builder.resources.%s: cpu and ram are required", name)
@@ -636,15 +685,30 @@ builder := func() {
 					"exec.builder.resources: onGPU was set without onCPU (GPU required), but this backend has no GPU support (exec.hasGpu is false). Add an onCPU block for an adaptive CPU fallback, or run on a GPU-enabled backend.")
 			}
 
+
+
+
+			setScratchFrom := func(block, ctx) {
+				if is_undefined(block.scratchFreeSpace) {
+					scratchFreeSpace = undefined
+					scratchFormulaAst = undefined
+					scratchFormulaFallback = undefined
+					return
+				}
+				setScratch(block.scratchFreeSpace, ctx)
+			}
+
 			if useGPU {
 				g := spec.onGPU
 				setCpu(g.cpu, "resources.onGPU")
 				setRam(g.ram, "resources.onGPU")
 				setVram(g.vram, "resources.onGPU")
+				setScratchFrom(g, "resources.onGPU")
 			} else {
 				c := spec.onCPU
 				setCpu(c.cpu, "resources.onCPU")
 				setRam(c.ram, "resources.onCPU")
+				setScratchFrom(c, "resources.onCPU")
 
 
 				gpuMemory = undefined
@@ -1277,6 +1341,53 @@ builder := func() {
 
 
 
+
+
+
+			tmpdirCompatNeeded := tmpdirCompat.isNeeded(
+				feats,
+				tmpdirCompat.isSizedRequest(scratchFreeSpace, scratchFormulaAst)
+			)
+			if tmpdirCompatNeeded {
+				filesToWrite[tmpdirCompat.SCRIPT_FILE_NAME] = tmpdirCompat.script()
+
+
+
+
+
+
+				args = slices.map(args, tmpdirCompat.rewriteScratchExpressions)
+				argsTyped = slices.map(argsTyped, func(a) {
+					a.value = tmpdirCompat.rewriteScratchExpressions(a.value)
+					return a
+				})
+				envs = maps.mapValues(envs, tmpdirCompat.rewriteScratchExpressions)
+				envsTyped = maps.mapValues(envsTyped, func(e) {
+					e.value = tmpdirCompat.rewriteScratchExpressions(e.value)
+					return e
+				})
+
+
+
+				if is_undefined(soft) {
+					wrapped := tmpdirCompat.wrapCommand(cmd)
+					cmd = wrapped.cmd
+
+
+
+
+					args = tmpdirCompat.concat(wrapped.prependArgs, args)
+					argsTyped = tmpdirCompat.concat(
+						slices.map(wrapped.prependArgs, func(a) {
+							return { type: execConstants.ARG_TYPE_STRING, value: a }
+						}),
+						argsTyped)
+				}
+			}
+
+
+
+
 			filesToWriteRefs := maps.mapValues(filesToWrite, func(v) {
 				if smart.isReference(v) { return v }
 				return smart.createValueResource(constants.RTYPE_BINARY_VALUE, v)
@@ -1333,10 +1444,12 @@ builder := func() {
 
 
 
-			hasFormula := !is_undefined(memFormulaAst) || !is_undefined(cpuFormulaAst) || !is_undefined(gpuFormulaAst)
+			hasFormula := !is_undefined(memFormulaAst) || !is_undefined(cpuFormulaAst) ||
+				!is_undefined(gpuFormulaAst) || !is_undefined(scratchFormulaAst)
 			resolvedMem := undefined
 			resolvedCpu := undefined
 			resolvedGpu := undefined
+			resolvedScratch := undefined
 			lineCountMapResource := undefined
 
 			if hasFormula {
@@ -1358,14 +1471,20 @@ builder := func() {
 					fb := execFormula.applyFormulaFallback({
 						ram: { ast: memFormulaAst, fallback: memFormulaFallback, static: ramRequest },
 						cpu: { ast: cpuFormulaAst, fallback: cpuFormulaFallback, static: cpuRequest },
-						gpu: { ast: gpuFormulaAst, fallback: gpuFormulaFallback, static: gpuMemory }
+						gpu: { ast: gpuFormulaAst, fallback: gpuFormulaFallback, static: gpuMemory },
+						scratch: { ast: scratchFormulaAst, fallback: scratchFormulaFallback, static: scratchFreeSpace }
 					})
 					ll.assert(is_undefined(fb.err), fb.err)
 					ramRequest = fb.ram
 					cpuRequest = fb.cpu
 					gpuMemory = fb.gpu
+
+
+
+					scratchFreeSpace = fb.scratch
 					if is_int(ramRequest) { ramRequest = string(ramRequest) + "b" }
 					if is_int(gpuMemory) { gpuMemory = string(gpuMemory) + "b" }
+					if is_int(scratchFreeSpace) { scratchFreeSpace = string(scratchFreeSpace) + "b" }
 				} else {
 					tagMap := {}
 					for fileName, tag in fileTags {
@@ -1390,6 +1509,11 @@ builder := func() {
 						refs := execFormula.referencedMetricFiles(resolvedGpu)
 						if !is_undefined(refs.lineCount) { sets.add(neededLineCount, refs.lineCount...) }
 					}
+					if !is_undefined(scratchFormulaAst) {
+						resolvedScratch = execFormula.resolveAst(scratchFormulaAst, tagMap, allFiles)
+						refs := execFormula.referencedMetricFiles(resolvedScratch)
+						if !is_undefined(refs.lineCount) { sets.add(neededLineCount, refs.lineCount...) }
+					}
 
 					if len(neededLineCount) > 0 {
 						perFile := {}
@@ -1419,6 +1543,24 @@ builder := func() {
 
 				if !is_undefined(gpuMemory) { quotaData.gpuMemory = gpuMemory }
 			}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+			scratchRequest := is_undefined(scratchFreeSpace) ? 0 : scratchFreeSpace
+			if is_int(scratchRequest) { scratchRequest = string(scratchRequest) + "b" }
+			quotaData.scratchFreeSpace = scratchRequest
 			metaExecInputs := {
 				quota: smart.createJsonResource(quotaData)
 			}
@@ -1442,6 +1584,7 @@ builder := func() {
 			if !is_undefined(resolvedMem) { metaExecInputs[execConstants.META_INPUT_MEM_FORMULA] = smart.createJsonResource(resolvedMem) }
 			if !is_undefined(resolvedCpu) { metaExecInputs[execConstants.META_INPUT_CPU_FORMULA] = smart.createJsonResource(resolvedCpu) }
 			if !is_undefined(resolvedGpu) { metaExecInputs[execConstants.META_INPUT_GPU_FORMULA] = smart.createJsonResource(resolvedGpu) }
+			if !is_undefined(resolvedScratch) { metaExecInputs[execConstants.META_INPUT_SCRATCH_FORMULA] = smart.createJsonResource(resolvedScratch) }
 			if !is_undefined(lineCountMapResource) { pureExecInputs[execConstants.REGULAR_INPUT_LINE_COUNT] = lineCountMapResource }
 
 
@@ -1466,10 +1609,14 @@ builder := func() {
 
 			if !is_undefined(soft) {
 				pureExecInputs.softwareInfo = soft.resource
+				descriptor := ll.fromStrict(soft.descriptor)
+				if tmpdirCompatNeeded {
+					descriptor = tmpdirCompat.patchSoftwareDescriptor(descriptor, feats.isDockerAvailable)
+				}
 				pureExecInputs.runOptions.software = {
 					name: soft.name,
 					version: soft.version,
-					descriptor: ll.fromStrict(soft.descriptor)
+					descriptor: descriptor
 				}
 			}
 

package/dist/tengo/lib/exec.limits.lib.tengo

@@ -56,6 +56,7 @@ quotaBuilder := func() {
 	cpu_request := undefined
 	ram_request := undefined
 	gpu_request := undefined
+	scratch_request := undefined
 
 	self = ll.toStrict({
 
@@ -127,6 +128,33 @@ quotaBuilder := func() {
 
 
 
+
+
+
+
+
+
+
+		scratchFreeSpace: func(amount) {
+			validation.assertType(
+				amount,
+				["or", "number", "string", "null"],
+				"exec.quota.builder.scratchFreeSpace: scratch space amount should be a number or string")
+
+
+
+			ll.assert(is_undefined(amount) || is_string(amount) || amount >= 0,
+				"exec.quota.builder.scratchFreeSpace: amount in bytes cannot be negative")
+			scratch_request = amount
+
+			return self
+		},
+
+
+
+
+
+
 		build: func() {
 			ll.assert(queue_name != "", "exec.quota.builder: queue name is mandatory and could not be empty string")
 
@@ -142,6 +170,9 @@ quotaBuilder := func() {
 			if !is_undefined(gpu_request) {
 				data.gpuMemory = gpu_request
 			}
+			if !is_undefined(scratch_request) {
+				data.scratchFreeSpace = scratch_request
+			}
 
 			return smart.ephemeralBuilder(_RTYPE_QUOTA, json.encode(data)).lockAndBuild()
 		}

package/dist/tengo/lib/exec.tmpdir-compat.lib.tengo

@@ -0,0 +1,270 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+text := import("text")
+
+
+
+
+
+
+
+SCRIPT_FILE_NAME := ".pl-run-compat-f9b7eca1b8.sh"
+
+
+
+
+
+
+
+
+
+SCRATCH_PATH_VARIABLE := "system.scratch.path"
+SCRATCH_SIZE_VARIABLES := ["system.scratch.bytes", "system.scratch.gib", "system.scratch.kib",
+	"system.scratch.mib", "system.scratch.gb", "system.scratch.kb", "system.scratch.mb",
+	"system.scratch.b"]
+
+
+
+
+WORKDIR_TMP_REL_PATH := ".pl/tmp"
+
+
+
+
+
+
+script := func() {
+	return `#!/bin/sh
+
+# Compatibility layer between a current workflow-tengo SDK and a Platforma Backend from before
+# the temporary-directory contract. Update the backend and this file stops appearing here.
+#
+# Written by exec.tmpdirCompat in @platforma-sdk/workflow-tengo.
+
+wd="${PL_JOB_WORKDIR:-$PWD}"
+tmp="$wd/` + WORKDIR_TMP_REL_PATH + `"
+mkdir -p "$tmp"
+
+TMPDIR="$tmp"
+TMP="$tmp"
+export TMPDIR TMP
+
+exec "$@"
+`
+}
+
+
+
+
+
+
+
+
+
+isNeeded := func(feats, scratchSized) {
+	if feats.supportsScratchSpaceRequests {
+
+		return false
+	}
+
+	if feats.hasScratchSpace {
+
+		return !scratchSized
+	}
+
+
+	return true
+}
+
+
+
+
+
+
+
+
+
+
+isSizedRequest := func(scratchFreeSpace, scratchFormulaAst) {
+	if !is_undefined(scratchFormulaAst) {
+		return true
+	}
+
+	if is_undefined(scratchFreeSpace) {
+		return false
+	}
+
+	return scratchFreeSpace != 0
+}
+
+
+
+
+isWindows := func() {
+	return ll.pathSeparator == "\\"
+}
+
+
+
+
+
+concat := func(prefix, tail) {
+	result := copy(prefix)
+	for item in tail {
+		result = append(result, item)
+	}
+
+	return result
+}
+
+
+
+
+_assertNotWindows := func() {
+	ll.assert(!isWindows(),
+		"exec.builder: this Platforma Backend is too old to provide a temporary directory (TMPDIR),"+
+			" and the workaround for it is POSIX-shell only, so it cannot run on Windows."+
+			" Update Platforma Backend to a version >=4.4.5")
+}
+
+
+
+
+
+
+
+
+
+
+
+
+patchSoftwareDescriptor := func(descriptor, isDockerAvailable) {
+	if isDockerAvailable && !is_undefined(descriptor.docker) {
+		docker := ll.fromStrict(descriptor.docker)
+
+		entrypoint := docker.entrypoint
+		if is_undefined(entrypoint) { entrypoint = [] }
+		docker.entrypoint = concat(["sh", "./" + SCRIPT_FILE_NAME], entrypoint)
+
+		descriptor.docker = docker
+		return descriptor
+	}
+
+
+
+	_assertNotWindows()
+
+	for kind in ["local", "binary"] {
+		if is_undefined(descriptor[kind]) { continue }
+
+		pkg := ll.fromStrict(descriptor[kind])
+		ll.assert(is_array(pkg.cmd) && len(pkg.cmd) != 0,
+			"exec.tmpdirCompat: software descriptor %q has no 'cmd' to wrap", kind)
+
+		pkg.cmd = concat(["sh", "./" + SCRIPT_FILE_NAME], pkg.cmd)
+		descriptor[kind] = pkg
+	}
+
+	return descriptor
+}
+
+
+
+
+
+
+wrapCommand := func(cmd) {
+	_assertNotWindows()
+
+	return {
+		cmd: "sh",
+		prependArgs: ["./" + SCRIPT_FILE_NAME, cmd]
+	}
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+rewriteScratchExpressions := func(value) {
+	if !is_string(value) {
+		return value
+	}
+
+
+	value = text.replace(value, "{" + SCRATCH_PATH_VARIABLE + "}", WORKDIR_TMP_REL_PATH, -1)
+	for variable in SCRATCH_SIZE_VARIABLES {
+		value = text.replace(value, "{" + variable + "}", "0", -1)
+	}
+
+	if !text.contains(value, "system.scratch.") {
+		return value
+	}
+
+
+	value = text.replace(value, SCRATCH_PATH_VARIABLE, "'" + WORKDIR_TMP_REL_PATH + "'", -1)
+	for variable in SCRATCH_SIZE_VARIABLES {
+		value = text.replace(value, variable, "0", -1)
+	}
+
+	return value
+}
+
+export ll.toStrict({
+	SCRIPT_FILE_NAME: SCRIPT_FILE_NAME,
+	WORKDIR_TMP_REL_PATH: WORKDIR_TMP_REL_PATH,
+	SCRATCH_PATH_VARIABLE: SCRATCH_PATH_VARIABLE,
+	SCRATCH_SIZE_VARIABLES: SCRATCH_SIZE_VARIABLES,
+
+	script: script,
+	isNeeded: isNeeded,
+	isSizedRequest: isSizedRequest,
+	isWindows: isWindows,
+	patchSoftwareDescriptor: patchSoftwareDescriptor,
+	wrapCommand: wrapCommand,
+	rewriteScratchExpressions: rewriteScratchExpressions,
+	concat: concat
+})

package/dist/tengo/lib/feats.lib.tengo

@@ -82,5 +82,17 @@ export ll.toStrict({
 
 
 
+
+
+	supportsScratchSpaceRequests: _isEnabled("supportsScratchSpaceRequests"),
+
+
+
+
+	hasScratchSpace:          _isEnabled("scratchSpace"),
+
+
+
+
 	multiStepProgress:        _isEnabled("multiStepProgress")
 })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "6.8.3",
+  "version": "6.10.0",
   "description": "Tengo SDK library",
   "type": "module",
   "types": "./dist/index.d.ts",
@@ -23,7 +23,7 @@
     "@platforma-open/milaboratories.software-ptexter": "1.2.4"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "4.0.23"
+    "@platforma-sdk/tengo-builder": "4.0.26"
   },
   "scripts": {
     "type-check": "pl-tengo check",