@platforma-sdk/workflow-tengo 5.7.3 → 5.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tengo/lib/annotations.lib.tengo +1 -1
- package/dist/tengo/lib/exec.constants.lib.tengo +8 -1
- package/dist/tengo/lib/exec.limits.lib.tengo +19 -13
- package/dist/tengo/lib/exec.runcmd.lib.tengo +2 -13
- package/dist/tengo/lib/feats.lib.tengo +15 -0
- package/dist/tengo/lib/slices.lib.tengo +20 -1
- package/dist/tengo/lib/workdir.lib.tengo +1 -1
- package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
- package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
- package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
- package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
- package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
- package/package.json +3 -3
|
@@ -50,7 +50,7 @@ computeAnnotations := func(inputs) {
|
|
|
50
50
|
|
|
51
51
|
columnsFromExpressions := util.extractColumnsFromExpressions(slices.map(steps, func(step) { return step.value }))
|
|
52
52
|
additionalColumns := is_array(inputs.additionalColumns) ? inputs.additionalColumns : []
|
|
53
|
-
columns := columnsFromExpressions + additionalColumns
|
|
53
|
+
columns := slices.unique(columnsFromExpressions + additionalColumns)
|
|
54
54
|
|
|
55
55
|
bundleBuilder := wf.createPBundleBuilder()
|
|
56
56
|
bundleBuilder.addAnchor("main", anchorRef)
|
|
@@ -11,6 +11,10 @@ MEDIUM_QUEUE := "medium"
|
|
|
11
11
|
LIGHT_QUEUE := "light"
|
|
12
12
|
UI_TASKS_QUEUE := "ui-tasks"
|
|
13
13
|
|
|
14
|
+
|
|
15
|
+
RUNNER_BATCH := "batch" // batch system execution (k8s jobs, etc.)
|
|
16
|
+
RUNNER_EXECUTOR := "executor" // local binary execution
|
|
17
|
+
|
|
14
18
|
export ll.toStrict({
|
|
15
19
|
ARG_TYPE_STRING: ARG_TYPE_STRING,
|
|
16
20
|
ARG_TYPE_EXPRESSION: ARG_TYPE_EXPRESSION,
|
|
@@ -21,5 +25,8 @@ export ll.toStrict({
|
|
|
21
25
|
UI_TASKS_QUEUE: UI_TASKS_QUEUE,
|
|
22
26
|
LIGHT_QUEUE: LIGHT_QUEUE,
|
|
23
27
|
MEDIUM_QUEUE: MEDIUM_QUEUE,
|
|
24
|
-
HEAVY_QUEUE: HEAVY_QUEUE
|
|
28
|
+
HEAVY_QUEUE: HEAVY_QUEUE,
|
|
29
|
+
|
|
30
|
+
RUNNER_BATCH: RUNNER_BATCH,
|
|
31
|
+
RUNNER_EXECUTOR: RUNNER_EXECUTOR
|
|
25
32
|
})
|
|
@@ -9,7 +9,6 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
|
9
9
|
|
|
10
10
|
oop := import("@platforma-sdk/workflow-tengo:oop")
|
|
11
11
|
validation := import("@platforma-sdk/workflow-tengo:validation")
|
|
12
|
-
slices := import("@platforma-sdk/workflow-tengo:slices")
|
|
13
12
|
smart := import("@platforma-sdk/workflow-tengo:smart")
|
|
14
13
|
json := import("json")
|
|
15
14
|
constants := import("@platforma-sdk/workflow-tengo:exec.constants")
|
|
@@ -23,8 +22,22 @@ _RTYPE_QUOTA := { Name: "Quota", Version: "1" }
|
|
|
23
22
|
|
|
24
23
|
|
|
25
24
|
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
getResourceTypes := func(queueName) {
|
|
26
|
+
|
|
27
|
+
result := feats.getResourceTypes(queueName)
|
|
28
|
+
if !is_undefined(result) {
|
|
29
|
+
return result
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
runnerType := constants.RUNNER_EXECUTOR
|
|
34
|
+
if feats.hasBatch && queueName != constants.UI_TASKS_QUEUE {
|
|
35
|
+
runnerType = constants.RUNNER_BATCH
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
runCommand: { Name: "RunCommand/" + runnerType, Version: "2" },
|
|
39
|
+
computeRequest: { Name: "ComputeRequest/" + runnerType, Version: "1" }
|
|
40
|
+
}
|
|
28
41
|
}
|
|
29
42
|
|
|
30
43
|
|
|
@@ -158,8 +171,6 @@ storageSpaceRequestBuilder := func() {
|
|
|
158
171
|
return self
|
|
159
172
|
}
|
|
160
173
|
|
|
161
|
-
_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1 := { Name: "ComputeRequest/executor", Version: "1" }
|
|
162
|
-
_RTYPE_COMPUTE_REQUEST_BATCH_V1 := { Name: "ComputeRequest/batch", Version: "1" }
|
|
163
174
|
|
|
164
175
|
_CR_FIELD_REQUEST := "request" // input
|
|
165
176
|
_CR_FIELD_ALLOCATION := "allocation" // output
|
|
@@ -205,13 +216,8 @@ computeRequestBuilder := func() {
|
|
|
205
216
|
ll.assert(smart.isReference(quota), "exec.limits.build: quota of ComputeRequest must be a valid reference")
|
|
206
217
|
ll.assert(queue_name != "", "exec.limits.build: queue name is required for computeRequestBuilder")
|
|
207
218
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
if useBatch(queue_name) {
|
|
211
|
-
requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_BATCH_V1)
|
|
212
|
-
} else {
|
|
213
|
-
requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1)
|
|
214
|
-
}
|
|
219
|
+
resourceTypes := getResourceTypes(queue_name)
|
|
220
|
+
requestBuilder := smart.ephemeralBuilder(resourceTypes.computeRequest)
|
|
215
221
|
|
|
216
222
|
requestBuilder.getField(_CR_FIELD_REQUEST).set(quota)
|
|
217
223
|
|
|
@@ -229,7 +235,7 @@ computeRequestBuilder := func() {
|
|
|
229
235
|
}
|
|
230
236
|
|
|
231
237
|
export ll.toStrict({
|
|
232
|
-
|
|
238
|
+
getResourceTypes: getResourceTypes,
|
|
233
239
|
quotaBuilder: quotaBuilder,
|
|
234
240
|
storageSpaceRequestBuilder: storageSpaceRequestBuilder,
|
|
235
241
|
computeRequestBuilder: computeRequestBuilder
|
|
@@ -19,8 +19,6 @@ limits := import("@platforma-sdk/workflow-tengo:exec.limits")
|
|
|
19
19
|
delayedCompAlloc := import("@platforma-sdk/workflow-tengo:exec.delayed-compute-allocation")
|
|
20
20
|
|
|
21
21
|
_RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
|
|
22
|
-
_RTYPE_RUN_COMMAND_EXEC_V2 := { Name: "RunCommand/executor", Version: "2" }
|
|
23
|
-
_RTYPE_RUN_COMMAND_BATCH_V2 := { Name: "RunCommand/batch", Version: "2" }
|
|
24
22
|
_RTYPE_RUN_COMMAND_CMD := { Name: "RunCommandCmd", Version: "1" }
|
|
25
23
|
_RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
|
|
26
24
|
_RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
|
|
@@ -669,17 +667,8 @@ builder := func(workdir, ...initialState) {
|
|
|
669
667
|
|
|
670
668
|
ll.assert(!is_undefined(queue), "runcmd.builder.build: queue is required for any command run")
|
|
671
669
|
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
if useBatch {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_BATCH_V2)
|
|
679
|
-
|
|
680
|
-
} else {
|
|
681
|
-
runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_EXEC_V2)
|
|
682
|
-
}
|
|
670
|
+
resourceTypes := limits.getResourceTypes(queue)
|
|
671
|
+
runRes = smart.ephemeralBuilder(resourceTypes.runCommand)
|
|
683
672
|
|
|
684
673
|
runRes.getField(_FIELD_WORKDIR_IN).set(workdir)
|
|
685
674
|
|
|
@@ -30,10 +30,25 @@ _hasRunner := func(e_id) {
|
|
|
30
30
|
return !is_undefined(_executors[e_id])
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
|
|
34
|
+
_hasGetResourceTypesFor := ll.methodExists(plapi, "getResourceTypesFor")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
_getResourceTypesFor := func(queueName) {
|
|
39
|
+
if !_hasGetResourceTypesFor {
|
|
40
|
+
return undefined
|
|
41
|
+
}
|
|
42
|
+
return plapi.getResourceTypesFor(queueName)
|
|
43
|
+
}
|
|
44
|
+
|
|
33
45
|
export ll.toStrict({
|
|
34
46
|
hasExec: _hasRunner("executor"), // local binary execution is available
|
|
35
47
|
hasBatch: _hasRunner("batch"), // batch system execution is available
|
|
36
48
|
|
|
49
|
+
|
|
50
|
+
getResourceTypes: _getResourceTypesFor,
|
|
51
|
+
|
|
37
52
|
pureFutureFields: _isEnabled("pureFutureFields") || _api >= 1, // 'pure' future fields magic that provides deduplication
|
|
38
53
|
commandExpressions: _isEnabled("commandExpressions") || _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )
|
|
39
54
|
|
|
@@ -376,6 +376,24 @@ sortUnique := func(list) {
|
|
|
376
376
|
return quickSortInPlace(uniqueList)
|
|
377
377
|
}
|
|
378
378
|
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
unique := func(list) {
|
|
386
|
+
arr := []
|
|
387
|
+
map := {}
|
|
388
|
+
for _, item in list {
|
|
389
|
+
if is_undefined(map[item]) {
|
|
390
|
+
arr = append(arr, item)
|
|
391
|
+
map[item] = true
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
return arr
|
|
395
|
+
}
|
|
396
|
+
|
|
379
397
|
export ll.toStrict({
|
|
380
398
|
length: length,
|
|
381
399
|
safeAppend: safeAppend,
|
|
@@ -396,5 +414,6 @@ export ll.toStrict({
|
|
|
396
414
|
fromSet: fromSet,
|
|
397
415
|
slice: slice,
|
|
398
416
|
normalize: normalize,
|
|
399
|
-
sortUnique: sortUnique
|
|
417
|
+
sortUnique: sortUnique,
|
|
418
|
+
unique: unique
|
|
400
419
|
})
|
|
@@ -323,7 +323,7 @@ _fill := func() {
|
|
|
323
323
|
quotaBuilder.ram(ram)
|
|
324
324
|
}
|
|
325
325
|
|
|
326
|
-
if limits.
|
|
326
|
+
if limits.getResourceTypes(queue).runCommand.Name == "RunCommand/" + execConstants.RUNNER_BATCH {
|
|
327
327
|
quotaBuilder.queue(execConstants.BATCH_QUEUE)
|
|
328
328
|
} else {
|
|
329
329
|
quotaBuilder.queue(queue)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platforma-sdk/workflow-tengo",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.8.1",
|
|
4
4
|
"description": "Tengo SDK library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -17,12 +17,12 @@
|
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
19
|
"@milaboratories/software-pframes-conv": "2.2.9",
|
|
20
|
-
"@platforma-open/milaboratories.software-small-binaries": "^
|
|
20
|
+
"@platforma-open/milaboratories.software-small-binaries": "^2.0.1",
|
|
21
21
|
"@platforma-open/milaboratories.software-ptabler": "1.14.0",
|
|
22
22
|
"@platforma-open/milaboratories.software-ptexter": "1.2.0"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
|
-
"@platforma-sdk/tengo-builder": "2.4.
|
|
25
|
+
"@platforma-sdk/tengo-builder": "2.4.11"
|
|
26
26
|
},
|
|
27
27
|
"scripts": {
|
|
28
28
|
"type-check": "pl-tengo check",
|