@platforma-sdk/workflow-tengo 3.0.3 → 3.2.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "ll.get-field" | "workflow.query-anchored" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "exec.R.create-renv" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "exec.python.create-venv" | "exec.monetization_to_json" | "exec.exec" | "exec.monetization_info" | "pframes.xsv-export-pframe" | "pframes.xsv-import-if-not-null" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
+declare type TplName = "ll.get-field" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "exec.R.create-renv" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "exec.python.create-venv" | "exec.monetization_to_json" | "exec.exec" | "exec.monetization_info" | "pframes.xsv-export-pframe" | "pframes.xsv-import-if-not-null" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.slice-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -1,12 +1,12 @@
 module.exports = { Templates: {
   'll.get-field': { type: 'from-file', path: require.resolve('./tengo/tpl/ll.get-field.plj.gz') },
-  'workflow.query-anchored': { type: 'from-file', path: require.resolve('./tengo/tpl/workflow.query-anchored.plj.gz') },
   'pframes.export-single-pcolumn': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
   'pframes.import-dir': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.import-dir.plj.gz') },
   'pframes.remove-nulls': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.remove-nulls.plj.gz') },
   'waiter.waiter': { type: 'from-file', path: require.resolve('./tengo/tpl/waiter.waiter.plj.gz') },
   'workdir.save': { type: 'from-file', path: require.resolve('./tengo/tpl/workdir.save.plj.gz') },
   'workflow.build-ctx': { type: 'from-file', path: require.resolve('./tengo/tpl/workflow.build-ctx.plj.gz') },
+  'workflow.query-anchored': { type: 'from-file', path: require.resolve('./tengo/tpl/workflow.query-anchored.plj.gz') },
   'exec.R.create-renv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.R.create-renv.plj.gz') },
   'exec.python.install-deps': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.install-deps.plj.gz') },
   'pframes.export-multiple-pcolumns': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
@@ -18,6 +18,7 @@ module.exports = { Templates: {
   'pframes.xsv-import-if-not-null': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
   'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') },
   'pframes.process-pcolumn-data': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
+  'pframes.slice-data': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.slice-data.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') },
   'pframes.export-pframe-for-ui': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe-for-ui.plj.gz') }
 }};

package/dist/index.mjs

@@ -1,13 +1,13 @@
 import { resolve } from 'node:path';
 export const Templates = {
   'll.get-field': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/ll.get-field.plj.gz') },
-  'workflow.query-anchored': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workflow.query-anchored.plj.gz') },
   'pframes.export-single-pcolumn': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
   'pframes.import-dir': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.import-dir.plj.gz') },
   'pframes.remove-nulls': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.remove-nulls.plj.gz') },
   'waiter.waiter': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/waiter.waiter.plj.gz') },
   'workdir.save': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workdir.save.plj.gz') },
   'workflow.build-ctx': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workflow.build-ctx.plj.gz') },
+  'workflow.query-anchored': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workflow.query-anchored.plj.gz') },
   'exec.R.create-renv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.R.create-renv.plj.gz') },
   'exec.python.install-deps': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.install-deps.plj.gz') },
   'pframes.export-multiple-pcolumns': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
@@ -19,6 +19,7 @@ export const Templates = {
   'pframes.xsv-import-if-not-null': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
   'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') },
   'pframes.process-pcolumn-data': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
+  'pframes.slice-data': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.slice-data.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') },
   'pframes.export-pframe-for-ui': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe-for-ui.plj.gz') }
 };

package/dist/tengo/lib/canonical.lib.tengo

@@ -0,0 +1,73 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+json := import("json")
+
+
+
+
+
+
+
+
+encode := func(obj) {
+    if is_map(obj) || ll.isStrict(obj) {
+
+        _obj := obj
+        if ll.isStrict(obj) {
+            _obj = ll.fromStrict(obj)
+        }
+
+
+        sortedKeys := maps.getKeys(_obj)
+
+
+        result := "{"
+
+        for i, key in sortedKeys {
+            value := _obj[key]
+
+
+            if i > 0 {
+                result = result + ","
+            }
+
+
+            result = result + json.encode(key) + ":"
+
+
+            if is_map(value) || is_array(value) || ll.isStrict(value) {
+                result = result + encode(value)
+            } else {
+                result = result + json.encode(value)
+            }
+        }
+
+        return result + "}"
+    } else if is_array(obj) {
+
+        result := "["
+
+        for i, value in obj {
+
+            if i > 0 {
+                result = result + ","
+            }
+
+
+            if is_map(value) || is_array(value) || ll.isStrict(value) {
+                result = result + encode(value)
+            } else {
+                result = result + json.encode(value)
+            }
+        }
+
+        return result + "]"
+    }
+
+
+    return json.encode(obj)
+}
+
+export ll.toStrict({
+    encode: encode
+})

package/dist/tengo/lib/exec.lib.tengo

@@ -4,6 +4,7 @@
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
 feats := import("@platforma-sdk/workflow-tengo:feats")
+validation := import("@platforma-sdk/workflow-tengo:validation")
 
 assets := import("@platforma-sdk/workflow-tengo:assets")
 smart := import("@platforma-sdk/workflow-tengo:smart")
@@ -15,7 +16,6 @@ path := import("@platforma-sdk/workflow-tengo:path")
 render := import("@platforma-sdk/workflow-tengo:render")
 enum := import("enum")
 oop := import("@platforma-sdk/workflow-tengo:oop")
-validation := import("@platforma-sdk/workflow-tengo:validation")
 constants := import("@platforma-sdk/workflow-tengo:constants")
 limits := import("@platforma-sdk/workflow-tengo:exec.limits")
 monetization := import("@platforma-sdk/workflow-tengo:exec.monetization_internal")
@@ -50,7 +50,6 @@ builder := func() {
 
 
 	queue := execConstants.HEAVY_QUEUE
-	hasLimits := false
 	cpuRequest := undefined
 	ramRequest := undefined
 
@@ -254,7 +253,6 @@ builder := func() {
 		cpu: func(amount) {
 			validation.assertType(amount, "number", "exec.builder.cpu: amount of cores must be a number")
 			ll.assert(amount > 0, "exec.builder.cpu: amount of cores should be greater than 0")
-			hasLimits = true
 			cpuRequest = amount
 
 			return self
@@ -274,7 +272,6 @@ builder := func() {
 		ram: func(amount) {
 			validation.assertType(amount, ["or", "number", "string"], "exec.builder.ram: RAM amount should be a number or string")
 			ll.assert(is_string(amount) || amount > 0, "exec.builder.ram: amount in bytes should be greater than 0")
-			hasLimits = true
 			ramRequest = amount
 
 			return self
@@ -728,11 +725,11 @@ builder := func() {
 			}
 
 			serviceExecInputs := {}
-			if hasLimits && feats.workdirLimits {
+			if feats.workdirLimits || feats.computeLimits {
 				serviceExecInputs.quota = limits.quotaBuilder().
 					queue(queue).
-					cpu(cpuRequest).
-					ram(ramRequest).
+					cpu(cpuRequest). // 'undefined' value is OK and makes runner to apply default queue limits
+					ram(ramRequest). // 'undefined' value is OK and makes runner to apply default queue limits
 					build()
 			}
 

package/dist/tengo/lib/exec.limits.lib.tengo

@@ -143,7 +143,69 @@ storageSpaceRequestBuilder := func() {
 	return self
 }
 
+_RTYPE_COMPUTE_REQUEST              := func(executor) { return { Name: "ComputeRequest/" + executor, Version: "1" } }
+_CR_DEFAULT_EXECUTOR_ID             := "executor"
+_CR_FIELD_REQUEST                   := "request"    // input
+_CR_FIELD_ALLOCATION                := "allocation" // output
+
+computeRequestBuilder := func() {
+	self := undefined
+
+	quota := undefined
+	executor := _CR_DEFAULT_EXECUTOR_ID
+
+	self = ll.toStrict({
+
+
+
+
+
+
+		quota: func(ref) {
+			validation.assertType(ref, validation.reference, "exec.limits.quota: quota must be a valid reference to Quota resource")
+			quota = ref
+
+			return self
+		},
+
+
+
+
+
+
+
+		executor: func(execID) {
+			validation.assertType(execID, "string", "exec.limits.executor: executor ID should be a string")
+			executor = execID
+
+			return self
+		},
+
+
+
+
+
+
+		build: func() {
+			ll.assert(smart.isReference(quota), "exec.limits.build: quota of ComputeRequest must be a valid reference")
+
+			rType := _RTYPE_COMPUTE_REQUEST(executor)
+			requestBuilder := smart.ephemeralBuilder(rType)
+			requestBuilder.getField(_CR_FIELD_REQUEST).set(quota)
+
+			return ll.toStrict(
+				oop.inherit(requestBuilder, {
+					allocation: requestBuilder.getField(_CR_FIELD_ALLOCATION)
+				})
+			)
+		}
+	})
+
+	return self
+}
+
 export ll.toStrict({
 	quotaBuilder:                    quotaBuilder,
-	storageSpaceRequestBuilder:      storageSpaceRequestBuilder
+	storageSpaceRequestBuilder:      storageSpaceRequestBuilder,
+	computeRequestBuilder:           computeRequestBuilder
 })

package/dist/tengo/lib/exec.monetization.lib.tengo

@@ -7,6 +7,8 @@ render := import("@platforma-sdk/workflow-tengo:render")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 validation := import("@platforma-sdk/workflow-tengo:validation")
 
+mznInfoTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:exec.monetization_info")
+
 
 
 
@@ -119,7 +121,7 @@ builder := func(productKey) {
 
 
 			result := render.create(
-				assets.importTemplate("@platforma-sdk/workflow-tengo:exec.monetization_info"),
+				mznInfoTpl,
 				{
 					date: date, // solely for deduplication
 					monetization: monetization.toJson(mnz),

package/dist/tengo/lib/exec.monetization_internal.lib.tengo

@@ -218,9 +218,10 @@ setUrl := func(mnz, url) {
 run := func(mnz, wd, queue) {
 	_validateIsReady(mnz)
 
+	mnzSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:mnz-client")
+
 	exprType := feats.commandExpressions ? constants.ARG_TYPE_EXPRESSION : constants.ARG_TYPE_VAR
 
-	mnzSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:mnz-client")
 	builder := runcmd.builder(wd)
 	builder.envTyped("MI_LICENSE", exprType, "{secrets.MI_LICENSE}")
 

package/dist/tengo/lib/exec.runcmd.lib.tengo

@@ -3,22 +3,33 @@
 
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
+validation := import("@platforma-sdk/workflow-tengo:validation")
 feats := import("@platforma-sdk/workflow-tengo:feats")
+oop := import("@platforma-sdk/workflow-tengo:oop")
 
 maps := import("@platforma-sdk/workflow-tengo:maps")
-oop := import("@platforma-sdk/workflow-tengo:oop")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 json := import("json")
 constants := import("@platforma-sdk/workflow-tengo:exec.constants")
 slices := import("@platforma-sdk/workflow-tengo:slices")
+limits := import("@platforma-sdk/workflow-tengo:exec.limits")
 
-_RTYPE_RUN_COMMAND := { Name: "RunCommand/executor", Version: "1" }
+_RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
+_RTYPE_RUN_COMMAND_V2 := { Name: "RunCommand/executor", Version: "2" }
 _RTYPE_RUN_COMMAND_CMD := { Name: "RunCommandCmd", Version: "1" }
 _RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
 _RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
 _RTYPE_RUN_COMMAND_REFS := { Name: "RunCommandRefs", Version: "1" }
 
+
+_FIELD_ALLOCATION := "allocation" // not in v1
+_FIELD_REFS := "refs"
+_FIELD_CMD := "cmd"
+_FIELD_ARGS := "args"
+_FIELD_OPTIONS := "options"
 _FIELD_WORKDIR_IN := "workdirIn"
+
+
 _FIELD_WORKDIR_OUT := "workdirOut"
 
 
@@ -66,7 +77,14 @@ _createExpressionArg := func(argTpl) {
 
 
 builder := func(workdir) {
-	res := smart.ephemeralBuilder(_RTYPE_RUN_COMMAND)
+	res := undefined
+
+	if feats.computeLimits {
+		res = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_V2)
+	} else {
+		res = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_V1)
+	}
+
 	res.getField(_FIELD_WORKDIR_IN).set(workdir)
 
 	self := undefined
@@ -76,6 +94,7 @@ builder := func(workdir) {
 	envs := {}
 	queue := ""
 	refs := {}
+	_allocation := undefined
 
 	substRules := {}
 	hasPatternArgs := false // TODO: deprecated, remove after a month of expressionRefs in prod.
@@ -92,6 +111,17 @@ builder := func(workdir) {
 
 
 
+		allocation: func(allocation) {
+			validation.assertType(allocation, validation.reference, "runcmd.builder: allocation should be a valid reference")
+			_allocation = allocation
+			return self
+		},
+
+
+
+
+
+
 		cmd: func(commandName) {
 			cmd = _createSimpleArg(commandName)
 			return self
@@ -366,11 +396,21 @@ builder := func(workdir) {
 				"runcmd.builder().build: expression arguments must be enabled only if platforma backend supports expressions, found: %v",
 				hasExpressionArgs)
 
+			if feats.computeLimits {
+				if !smart.isReference(_allocation) {
+
+
+					_quota := limits.quotaBuilder().queue(queue).build()
+					_allocation = limits.computeRequestBuilder().quota(_quota).build().allocation
+				}
+				self.getField(_FIELD_ALLOCATION).set(_allocation)
+			}
+
 			refsRes := smart.createMapResourceWithType(_RTYPE_RUN_COMMAND_REFS, refs)
-			self.getField("refs").set(refsRes)
+			self.getField(_FIELD_REFS).set(refsRes)
 
 			cmdRes := smart.createValueResource(_RTYPE_RUN_COMMAND_CMD, json.encode(cmd))
-			self.getField("cmd").set(cmdRes)
+			self.getField(_FIELD_CMD).set(cmdRes)
 
 			options := {
 				queueName: queue,
@@ -383,10 +423,10 @@ builder := func(workdir) {
 			}
 
 			optionsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_OPTIONS, json.encode(options))
-			self.getField("options").set(optionsRes)
+			self.getField(_FIELD_OPTIONS).set(optionsRes)
 
 			argsRes := smart.createValueResource(_RTYPE_RUN_COMMAND_ARGS, json.encode(args))
-			self.getField("args").set(argsRes)
+			self.getField(_FIELD_ARGS).set(argsRes)
 
 			res := ll.toStrict(oop.inherit(self.lockAndBuild(), {
 				"workdir": self.outputs()[_FIELD_WORKDIR_OUT],

package/dist/tengo/lib/feats.lib.tengo

@@ -6,11 +6,12 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 
 _api := plapi.apiVersion
 if is_undefined(_api) {
-    _api = 0
+	_api = 0
 }
 
 export ll.toStrict({
-    pureFutureFields:           _api >= 1,
-    commandExpressions:         _api >= 2,
-    workdirLimits:              _api >= 4
+	pureFutureFields:           _api >= 1, // 'pure' future fields magic that provides deduplication
+	commandExpressions:         _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )
+	workdirLimits:              _api >= 4, // workdir storage controller can limit concurrency
+	computeLimits:              _api >= 8  // runner can plan CPU and RAM
 })

package/dist/tengo/lib/maps.lib.tengo

@@ -13,7 +13,7 @@ getKeys := func(map) {
 		res = append(res, k)
 	}
 
-	return slices.quickSort(res)
+	return slices.quickSortInPlace(res)
 }
 
 

package/dist/tengo/lib/pframes.builder.lib.tengo

@@ -2,7 +2,8 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 oop := import("@platforma-sdk/workflow-tengo:oop")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 pConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
-
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+validation := import("@platforma-sdk/workflow-tengo:validation")
 
 
 
@@ -15,6 +16,9 @@ pFrameBuilder := func() {
 
 		add: func(column, spec, data) {
 			if (!is_undefined(spec)) {
+				if !smart.isReference(spec) {
+					validation.assertType(spec, pSpec.P_OBJECT_SPEC_SCHEMA)
+				}
 				r.createInputField(column + ".spec").setRefOrJson(spec)
 			}
 			if (!is_undefined(data)) {