@platforma-sdk/workflow-tengo 5.7.3 → 5.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/tengo/lib/annotations.lib.tengo +1 -1
  2. package/dist/tengo/lib/exec.constants.lib.tengo +8 -1
  3. package/dist/tengo/lib/exec.limits.lib.tengo +19 -13
  4. package/dist/tengo/lib/exec.runcmd.lib.tengo +2 -13
  5. package/dist/tengo/lib/feats.lib.tengo +15 -0
  6. package/dist/tengo/lib/slices.lib.tengo +20 -1
  7. package/dist/tengo/lib/workdir.lib.tengo +1 -1
  8. package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
  9. package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
  10. package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
  11. package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
  12. package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
  13. package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
  14. package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
  15. package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
  16. package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
  17. package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
  18. package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
  19. package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
  20. package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
  21. package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
  22. package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
  23. package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
  24. package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
  25. package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
  26. package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
  27. package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
  28. package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
  29. package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
  30. package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
  31. package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
  32. package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
  33. package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
  34. package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
  35. package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
  36. package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
  37. package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
  38. package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
  39. package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
  40. package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
  41. package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
  42. package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
  43. package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
  44. package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
  45. package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
  46. package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
  47. package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
  48. package/package.json +3 -3
@@ -50,7 +50,7 @@ computeAnnotations := func(inputs) {
50
50
 
51
51
  columnsFromExpressions := util.extractColumnsFromExpressions(slices.map(steps, func(step) { return step.value }))
52
52
  additionalColumns := is_array(inputs.additionalColumns) ? inputs.additionalColumns : []
53
- columns := columnsFromExpressions + additionalColumns
53
+ columns := slices.unique(columnsFromExpressions + additionalColumns)
54
54
 
55
55
  bundleBuilder := wf.createPBundleBuilder()
56
56
  bundleBuilder.addAnchor("main", anchorRef)
@@ -11,6 +11,10 @@ MEDIUM_QUEUE := "medium"
11
11
  LIGHT_QUEUE := "light"
12
12
  UI_TASKS_QUEUE := "ui-tasks"
13
13
 
14
+
15
+ RUNNER_BATCH := "batch" // batch system execution (k8s jobs, etc.)
16
+ RUNNER_EXECUTOR := "executor" // local binary execution
17
+
14
18
  export ll.toStrict({
15
19
  ARG_TYPE_STRING: ARG_TYPE_STRING,
16
20
  ARG_TYPE_EXPRESSION: ARG_TYPE_EXPRESSION,
@@ -21,5 +25,8 @@ export ll.toStrict({
21
25
  UI_TASKS_QUEUE: UI_TASKS_QUEUE,
22
26
  LIGHT_QUEUE: LIGHT_QUEUE,
23
27
  MEDIUM_QUEUE: MEDIUM_QUEUE,
24
- HEAVY_QUEUE: HEAVY_QUEUE
28
+ HEAVY_QUEUE: HEAVY_QUEUE,
29
+
30
+ RUNNER_BATCH: RUNNER_BATCH,
31
+ RUNNER_EXECUTOR: RUNNER_EXECUTOR
25
32
  })
@@ -9,7 +9,6 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
9
9
 
10
10
  oop := import("@platforma-sdk/workflow-tengo:oop")
11
11
  validation := import("@platforma-sdk/workflow-tengo:validation")
12
- slices := import("@platforma-sdk/workflow-tengo:slices")
13
12
  smart := import("@platforma-sdk/workflow-tengo:smart")
14
13
  json := import("json")
15
14
  constants := import("@platforma-sdk/workflow-tengo:exec.constants")
@@ -23,8 +22,22 @@ _RTYPE_QUOTA := { Name: "Quota", Version: "1" }
23
22
 
24
23
 
25
24
 
26
- useBatch := func(queueName) {
27
- return feats.hasBatch && !slices.hasElement([ constants.UI_TASKS_QUEUE ], queueName)
25
+ getResourceTypes := func(queueName) {
26
+
27
+ result := feats.getResourceTypes(queueName)
28
+ if !is_undefined(result) {
29
+ return result
30
+ }
31
+
32
+
33
+ runnerType := constants.RUNNER_EXECUTOR
34
+ if feats.hasBatch && queueName != constants.UI_TASKS_QUEUE {
35
+ runnerType = constants.RUNNER_BATCH
36
+ }
37
+ return {
38
+ runCommand: { Name: "RunCommand/" + runnerType, Version: "2" },
39
+ computeRequest: { Name: "ComputeRequest/" + runnerType, Version: "1" }
40
+ }
28
41
  }
29
42
 
30
43
 
@@ -158,8 +171,6 @@ storageSpaceRequestBuilder := func() {
158
171
  return self
159
172
  }
160
173
 
161
- _RTYPE_COMPUTE_REQUEST_EXECUTOR_V1 := { Name: "ComputeRequest/executor", Version: "1" }
162
- _RTYPE_COMPUTE_REQUEST_BATCH_V1 := { Name: "ComputeRequest/batch", Version: "1" }
163
174
 
164
175
  _CR_FIELD_REQUEST := "request" // input
165
176
  _CR_FIELD_ALLOCATION := "allocation" // output
@@ -205,13 +216,8 @@ computeRequestBuilder := func() {
205
216
  ll.assert(smart.isReference(quota), "exec.limits.build: quota of ComputeRequest must be a valid reference")
206
217
  ll.assert(queue_name != "", "exec.limits.build: queue name is required for computeRequestBuilder")
207
218
 
208
- requestBuilder := undefined
209
-
210
- if useBatch(queue_name) {
211
- requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_BATCH_V1)
212
- } else {
213
- requestBuilder = smart.ephemeralBuilder(_RTYPE_COMPUTE_REQUEST_EXECUTOR_V1)
214
- }
219
+ resourceTypes := getResourceTypes(queue_name)
220
+ requestBuilder := smart.ephemeralBuilder(resourceTypes.computeRequest)
215
221
 
216
222
  requestBuilder.getField(_CR_FIELD_REQUEST).set(quota)
217
223
 
@@ -229,7 +235,7 @@ computeRequestBuilder := func() {
229
235
  }
230
236
 
231
237
  export ll.toStrict({
232
- useBatch: useBatch,
238
+ getResourceTypes: getResourceTypes,
233
239
  quotaBuilder: quotaBuilder,
234
240
  storageSpaceRequestBuilder: storageSpaceRequestBuilder,
235
241
  computeRequestBuilder: computeRequestBuilder
@@ -19,8 +19,6 @@ limits := import("@platforma-sdk/workflow-tengo:exec.limits")
19
19
  delayedCompAlloc := import("@platforma-sdk/workflow-tengo:exec.delayed-compute-allocation")
20
20
 
21
21
  _RTYPE_RUN_COMMAND_V1 := { Name: "RunCommand/executor", Version: "1" }
22
- _RTYPE_RUN_COMMAND_EXEC_V2 := { Name: "RunCommand/executor", Version: "2" }
23
- _RTYPE_RUN_COMMAND_BATCH_V2 := { Name: "RunCommand/batch", Version: "2" }
24
22
  _RTYPE_RUN_COMMAND_CMD := { Name: "RunCommandCmd", Version: "1" }
25
23
  _RTYPE_RUN_COMMAND_ARGS := { Name: "RunCommandArgs", Version: "1" }
26
24
  _RTYPE_RUN_COMMAND_OPTIONS := { Name: "run-command/options", Version: "1" }
@@ -669,17 +667,8 @@ builder := func(workdir, ...initialState) {
669
667
 
670
668
  ll.assert(!is_undefined(queue), "runcmd.builder.build: queue is required for any command run")
671
669
 
672
- useBatch := limits.useBatch(queue)
673
-
674
- if useBatch {
675
-
676
-
677
-
678
- runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_BATCH_V2)
679
-
680
- } else {
681
- runRes = smart.ephemeralBuilder(_RTYPE_RUN_COMMAND_EXEC_V2)
682
- }
670
+ resourceTypes := limits.getResourceTypes(queue)
671
+ runRes = smart.ephemeralBuilder(resourceTypes.runCommand)
683
672
 
684
673
  runRes.getField(_FIELD_WORKDIR_IN).set(workdir)
685
674
 
@@ -30,10 +30,25 @@ _hasRunner := func(e_id) {
30
30
  return !is_undefined(_executors[e_id])
31
31
  }
32
32
 
33
+
34
+ _hasGetResourceTypesFor := ll.methodExists(plapi, "getResourceTypesFor")
35
+
36
+
37
+
38
+ _getResourceTypesFor := func(queueName) {
39
+ if !_hasGetResourceTypesFor {
40
+ return undefined
41
+ }
42
+ return plapi.getResourceTypesFor(queueName)
43
+ }
44
+
33
45
  export ll.toStrict({
34
46
  hasExec: _hasRunner("executor"), // local binary execution is available
35
47
  hasBatch: _hasRunner("batch"), // batch system execution is available
36
48
 
49
+
50
+ getResourceTypes: _getResourceTypesFor,
51
+
37
52
  pureFutureFields: _isEnabled("pureFutureFields") || _api >= 1, // 'pure' future fields magic that provides deduplication
38
53
  commandExpressions: _isEnabled("commandExpressions") || _api >= 2, // rich expressions in command arguments ( {{ system.ram_bytes / 20 }} )
39
54
 
@@ -376,6 +376,24 @@ sortUnique := func(list) {
376
376
  return quickSortInPlace(uniqueList)
377
377
  }
378
378
 
379
+
380
+
381
+
382
+
383
+
384
+
385
+ unique := func(list) {
386
+ arr := []
387
+ map := {}
388
+ for _, item in list {
389
+ if is_undefined(map[item]) {
390
+ arr = append(arr, item)
391
+ map[item] = true
392
+ }
393
+ }
394
+ return arr
395
+ }
396
+
379
397
  export ll.toStrict({
380
398
  length: length,
381
399
  safeAppend: safeAppend,
@@ -396,5 +414,6 @@ export ll.toStrict({
396
414
  fromSet: fromSet,
397
415
  slice: slice,
398
416
  normalize: normalize,
399
- sortUnique: sortUnique
417
+ sortUnique: sortUnique,
418
+ unique: unique
400
419
  })
@@ -323,7 +323,7 @@ _fill := func() {
323
323
  quotaBuilder.ram(ram)
324
324
  }
325
325
 
326
- if limits.useBatch(queue) {
326
+ if limits.getResourceTypes(queue).runCommand.Name == "RunCommand/" + execConstants.RUNNER_BATCH {
327
327
  quotaBuilder.queue(execConstants.BATCH_QUEUE)
328
328
  } else {
329
329
  quotaBuilder.queue(queue)
Binary file
Binary file
Binary file
Binary file
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platforma-sdk/workflow-tengo",
3
- "version": "5.7.3",
3
+ "version": "5.8.1",
4
4
  "description": "Tengo SDK library",
5
5
  "type": "module",
6
6
  "types": "./dist/index.d.ts",
@@ -17,12 +17,12 @@
17
17
  ],
18
18
  "dependencies": {
19
19
  "@milaboratories/software-pframes-conv": "2.2.9",
20
- "@platforma-open/milaboratories.software-small-binaries": "^1.15.25",
20
+ "@platforma-open/milaboratories.software-small-binaries": "^2.0.1",
21
21
  "@platforma-open/milaboratories.software-ptabler": "1.14.0",
22
22
  "@platforma-open/milaboratories.software-ptexter": "1.2.0"
23
23
  },
24
24
  "devDependencies": {
25
- "@platforma-sdk/tengo-builder": "2.4.2"
25
+ "@platforma-sdk/tengo-builder": "2.4.11"
26
26
  },
27
27
  "scripts": {
28
28
  "type-check": "pl-tengo check",