@platforma-sdk/workflow-tengo 5.7.3 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tengo/lib/annotations.lib.tengo +1 -1
- package/dist/tengo/lib/exec.constants.lib.tengo +8 -1
- package/dist/tengo/lib/exec.limits.lib.tengo +18 -12
- package/dist/tengo/lib/exec.runcmd.lib.tengo +6 -13
- package/dist/tengo/lib/feats.lib.tengo +15 -0
- package/dist/tengo/lib/slices.lib.tengo +20 -1
- package/dist/tengo/lib/workdir.lib.tengo +1 -1
- package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
- package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
- package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
- package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
- package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
- package/package.json +5 -5
|
@@ -50,7 +50,7 @@ computeAnnotations := func(inputs) {
|
|
|
50
50
|
|
|
51
51
|
columnsFromExpressions := util.extractColumnsFromExpressions(slices.map(steps, func(step) { return step.value }))
|
|
52
52
|
additionalColumns := is_array(inputs.additionalColumns) ? inputs.additionalColumns : []
|
|
53
|
-
columns := columnsFromExpressions + additionalColumns
|
|
53
|
+
columns := slices.unique(columnsFromExpressions + additionalColumns)
|
|
54
54
|
|
|
55
55
|
bundleBuilder := wf.createPBundleBuilder()
|
|
56
56
|
bundleBuilder.addAnchor("main", anchorRef)
|
|
@@ -11,6 +11,10 @@ MEDIUM_QUEUE := "medium"
|
|
|
11
11
|
LIGHT_QUEUE := "light"
|
|
12
12
|
UI_TASKS_QUEUE := "ui-tasks"
|
|
13
13
|
|
|
14
|
+
|
|
15
|
+
RUNNER_BATCH := "batch" // batch system execution (k8s jobs, etc.)
|
|
16
|
+
RUNNER_EXECUTOR := "executor" // local binary execution
|
|
17
|
+
|
|
14
18
|
export ll.toStrict({
|
|
15
19
|
ARG_TYPE_STRING: ARG_TYPE_STRING,
|
|
16
20
|
ARG_TYPE_EXPRESSION: ARG_TYPE_EXPRESSION,
|
|
@@ -21,5 +25,8 @@ export ll.toStrict({
|
|
|
21
25
|
UI_TASKS_QUEUE: UI_TASKS_QUEUE,
|
|
22
26
|
LIGHT_QUEUE: LIGHT_QUEUE,
|
|
23
27
|
MEDIUM_QUEUE: MEDIUM_QUEUE,
|
|
24
|
-
HEAVY_QUEUE: HEAVY_QUEUE
|
|
28
|
+
HEAVY_QUEUE: HEAVY_QUEUE,
|
|
29
|
+
|
|
30
|
+
RUNNER_BATCH: RUNNER_BATCH,
|
|
31
|
+
RUNNER_EXECUTOR: RUNNER_EXECUTOR
|
|
25
32
|
})
|
|
@@ -23,8 +23,18 @@ _RTYPE_QUOTA := { Name: "Quota", Version: "1" }
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
getRunnerType := func(queueName) {
|
|
27
|
+
|
|
28
|
+
runner := feats.getRunnerFor(queueName)
|
|
29
|
+
if !is_undefined(runner) {
|
|
30
|
+
return runner
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
if feats.hasBatch && !slices.hasElement([ constants.UI_TASKS_QUEUE ], queueName) {
|
|
35
|
+
return constants.RUNNER_BATCH
|
|
36
|
+
}
|
|
37
|
+
return constants.RUNNER_EXECUTOR
|
|
28
38
|
}
|
|
29
39
|
|
|
30
40
|
|
|
@@ -158,8 +168,9 @@ storageSpaceRequestBuilder := func() {
|
|
|
158
168
|
return self
|
|
159
169
|
}
|
|
160
170
|
|
|
161
|
-
|
|
162
|
-
|
|
171
|
+
_computeRequestResourceType := func(runnerType) {
|
|
172
|
+
return { Name: "ComputeRequest/" + runnerType, Version: "1" }
|
|
173
|
+
}
|
|
163
174
|
|
|
164
175
|
_CR_FIELD_REQUEST := "request" // input
|
|
165
176
|
_CR_FIELD_ALLOCATION := "allocation" // output
|
|
@@ -205,13 +216,8 @@ computeRequestBuilder := func() {
|
|
|
205
216
|
ll.assert(smart.isReference(quota), "exec.limits.build: quota of ComputeRequest must be a valid reference")
|
|
206
217
|
ll.assert(queue_name != "", "exec.limits.build: queue name is required for computeRequestBuilder")
|
|
207
218
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
if useBatch(queue_name) {
|
|
211
|
-
requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_BATCH_V1)
|
|
212
|
-
} else {
|
|
213
|
-
requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1)
|
|
214
|
-
}
|
|
219
|
+
runnerType := getRunnerType(queue_name)
|
|
220
|
+
requestBuilder := smart.ephemeralBuilder(_computeRequestResourceType(runnerType))
|
|
215
221
|
|
|
216
222
|
requestBuilder.getField(_CR_FIELD_REQUEST).set(quota)
|
|
217
223
|
|
|
@@ -229,7 +235,7 @@ computeRequestBuilder := func() {
|
|
|
229
235
|
}
|
|
230
236
|
|
|
231
237
|
export ll.toStrict({
|
|
232
|
-
|
|
238
|
+
getRunnerType: getRunnerType,
|
|
233
239
|
quotaBuilder: quotaBuilder,
|
|
234
240
|
storageSpaceRequestBuilder: storageSpaceRequestBuilder,
|
|
235
241
|
computeRequestBuilder: computeRequestBuilder
|
|
@@ -19,8 +19,10 @@ limits := import("@platforma-sdk/workflow-tengo:exec.limits")
|
|
|
19
19
|
delayedCompAlloc := import("@platforma-sdk/workflow-tengo:exec.delayed-compute-allocation")
|
|
20
20
|
|
|
21
21
|
_RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
|
|
23
|
+
_runCommandResourceType := func(runnerType) {
|
|
24
|
+
return { Name: "RunCommand/" + runnerType, Version: "2" }
|
|
25
|
+
}
|
|
24
26
|
_RTYPE_RUN_COMMAND_CMD := { Name: "RunCommandCmd", Version: "1" }
|
|
25
27
|
_RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
|
|
26
28
|
_RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
|
|
@@ -669,17 +671,8 @@ builder := func(workdir, ...initialState) {
|
|
|
669
671
|
|
|
670
672
|
ll.assert(!is_undefined(queue), "runcmd.builder.build: queue is required for any command run")
|
|
671
673
|
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
if useBatch {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_BATCH_V2)
|
|
679
|
-
|
|
680
|
-
} else {
|
|
681
|
-
runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_EXEC_V2)
|
|
682
|
-
}
|
|
674
|
+
runnerType := limits.getRunnerType(queue)
|
|
675
|
+
runRes = smart.ephemeralBuilder(_runCommandResourceType(runnerType))
|
|
683
676
|
|
|
684
677
|
runRes.getField(_FIELD_WORKDIR_IN).set(workdir)
|
|
685
678
|
|
|
@@ -30,10 +30,25 @@ _hasRunner := func(e_id) {
|
|
|
30
30
|
return !is_undefined(_executors[e_id])
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
|
|
34
|
+
_hasGetRunnerFor := ll.methodExists(plapi, "getRunnerFor")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
_getRunnerFor := func(queueName) {
|
|
39
|
+
if !_hasGetRunnerFor {
|
|
40
|
+
return undefined
|
|
41
|
+
}
|
|
42
|
+
return plapi.getRunnerFor(queueName)
|
|
43
|
+
}
|
|
44
|
+
|
|
33
45
|
export ll.toStrict({
|
|
34
46
|
hasExec: _hasRunner("executor"), // local binary execution is available
|
|
35
47
|
hasBatch: _hasRunner("batch"), // batch system execution is available
|
|
36
48
|
|
|
49
|
+
|
|
50
|
+
getRunnerFor: _getRunnerFor,
|
|
51
|
+
|
|
37
52
|
pureFutureFields: _isEnabled("pureFutureFields") || _api >= 1, // 'pure' future fields magic that provides deduplication
|
|
38
53
|
commandExpressions: _isEnabled("commandExpressions") || _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )
|
|
39
54
|
|
|
@@ -376,6 +376,24 @@ sortUnique := func(list) {
|
|
|
376
376
|
return quickSortInPlace(uniqueList)
|
|
377
377
|
}
|
|
378
378
|
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
unique := func(list) {
|
|
386
|
+
arr := []
|
|
387
|
+
map := {}
|
|
388
|
+
for _, item in list {
|
|
389
|
+
if is_undefined(map[item]) {
|
|
390
|
+
arr = append(arr, item)
|
|
391
|
+
map[item] = true
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
return arr
|
|
395
|
+
}
|
|
396
|
+
|
|
379
397
|
export ll.toStrict({
|
|
380
398
|
length: length,
|
|
381
399
|
safeAppend: safeAppend,
|
|
@@ -396,5 +414,6 @@ export ll.toStrict({
|
|
|
396
414
|
fromSet: fromSet,
|
|
397
415
|
slice: slice,
|
|
398
416
|
normalize: normalize,
|
|
399
|
-
sortUnique: sortUnique
|
|
417
|
+
sortUnique: sortUnique,
|
|
418
|
+
unique: unique
|
|
400
419
|
})
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platforma-sdk/workflow-tengo",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.8.0",
|
|
4
4
|
"description": "Tengo SDK library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -17,12 +17,12 @@
|
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
19
|
"@milaboratories/software-pframes-conv": "2.2.9",
|
|
20
|
-
"@platforma-open/milaboratories.software-small-binaries": "^1.15.
|
|
21
|
-
"@platforma-open/milaboratories.software-
|
|
22
|
-
"@platforma-open/milaboratories.software-
|
|
20
|
+
"@platforma-open/milaboratories.software-small-binaries": "^1.15.26",
|
|
21
|
+
"@platforma-open/milaboratories.software-ptexter": "1.2.0",
|
|
22
|
+
"@platforma-open/milaboratories.software-ptabler": "1.14.0"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
|
-
"@platforma-sdk/tengo-builder": "2.4.
|
|
25
|
+
"@platforma-sdk/tengo-builder": "2.4.8"
|
|
26
26
|
},
|
|
27
27
|
"scripts": {
|
|
28
28
|
"type-check": "pl-tengo check",
|