@platforma-sdk/workflow-tengo 6.11.1 → 6.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/tengo/lib/exec.formula.lib.tengo +16 -0
  2. package/dist/tengo/lib/pt.lib.tengo +21 -0
  3. package/dist/tengo/lib/pt.sizing.lib.tengo +53 -1
  4. package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
  5. package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
  6. package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
  7. package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
  8. package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
  9. package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
  10. package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
  11. package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
  12. package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
  13. package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
  14. package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
  15. package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
  16. package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
  17. package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
  18. package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
  19. package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
  20. package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
  21. package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
  22. package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
  23. package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
  24. package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
  25. package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
  26. package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
  27. package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
  28. package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
  29. package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
  30. package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
  31. package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
  32. package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
  33. package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
  34. package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
  35. package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
  36. package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
  37. package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
  38. package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
  39. package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
  40. package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
  41. package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
  42. package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
  43. package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
  44. package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
  45. package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
  46. package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
  47. package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
  48. package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
  49. package/package.json +3 -3
@@ -210,6 +210,21 @@ evaluateResource := func(node, resolver, label) {
210
210
 
211
211
 
212
212
 
213
+ evaluateScratch := func(node, resolver) {
214
+ node = _unwrap(node)
215
+ ll.assert(!sets.hasElement(_BOOL_OPS, node.op),
216
+ "exec resource formula (scratchFreeSpace) must compute a number, but its top-level op is %q (a comparison/logic op). Use comparisons and logic only inside f.when(...).",
217
+ node.op)
218
+ v := evaluate(node, resolver)
219
+ ll.assert(is_int(v) && v >= 0,
220
+ "exec resource formula (scratchFreeSpace) must evaluate to a non-negative integer, got %v. Floor a subtraction that can underflow with .atLeast(0).",
221
+ v)
222
+ return v
223
+ }
224
+
225
+
226
+
227
+
213
228
 
214
229
 
215
230
 
@@ -450,6 +465,7 @@ export {
450
465
  isFormula: func(x) { return is_map(x) && (!is_undefined(x.__isFormula__) || !is_undefined(x.op)) },
451
466
 
452
467
  evaluateResource: evaluateResource,
468
+ evaluateScratch: evaluateScratch,
453
469
  resolveAst: resolveAst,
454
470
  referencedMetricFiles: referencedMetricFiles,
455
471
  applyFormulaFallback: applyFormulaFallback,
@@ -412,6 +412,7 @@ workflow := func() {
412
412
  cpu := undefined
413
413
  mem := undefined
414
414
  memFloor := undefined
415
+ scratchFreeSpace := undefined
415
416
  inputCache := undefined
416
417
  queue := execConstants.MEDIUM_QUEUE
417
418
 
@@ -931,6 +932,25 @@ workflow := func() {
931
932
 
932
933
 
933
934
 
935
+
936
+
937
+
938
+ scratchFreeSpace: func(amount) {
939
+ ll.assert(is_string(amount) || (is_int(amount) && (amount == 0 || amount >= 16 * units.MiB)),
940
+ "pt.workflow.scratchFreeSpace: expected 0, at least 16 MiB in bytes, or a size string, got %v", amount)
941
+ scratchFreeSpace = amount
942
+ return self
943
+ },
944
+
945
+
946
+
947
+
948
+
949
+
950
+
951
+
952
+
953
+
934
954
  cacheInputs: func(time) {
935
955
  ll.assert(
936
956
  is_int(time),
@@ -1031,6 +1051,7 @@ workflow := func() {
1031
1051
  cpu: cpu,
1032
1052
  mem: mem,
1033
1053
  memFloor: memFloor,
1054
+ scratchFreeSpace: scratchFreeSpace,
1034
1055
  inputCache: inputCache
1035
1056
  }
1036
1057
  ptResult := render.create(workflowRunTpl, runInputs, {
@@ -18,6 +18,30 @@
18
18
 
19
19
 
20
20
 
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
21
45
 
22
46
 
23
47
 
@@ -33,6 +57,11 @@ RAM_FLOOR_GIB := 2
33
57
  RAM_CAP_GIB := 256
34
58
  RAM_STATIC_FALLBACK := "4GiB"
35
59
 
60
+ SCRATCH_PER_READ_FRAME := 16
61
+ SCRATCH_SORT := 16
62
+ SCRATCH_PER_WRITE_FRAME := 2
63
+ SCRATCH_DEVICE_MIN_GIB := 16
64
+
36
65
  CPU_MIN := 2
37
66
  CPU_MAX := 8
38
67
  CPU_GIB_PER_CORE := 16
@@ -76,9 +105,28 @@ cpuFormula := func() {
76
105
  staticFallback(CPU_MIN)
77
106
  }
78
107
 
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+ scratchFormula := func(readFrames, writeFrames) {
116
+ ll.assert(is_int(readFrames) && readFrames >= 0 && is_int(writeFrames) && writeFrames >= 0,
117
+ "pt.sizing: frame counts must be non-negative integers, got %v and %v", readFrames, writeFrames)
118
+ if readFrames == 0 && writeFrames == 0 {
119
+ return 0
120
+ }
121
+ factor := SCRATCH_PER_READ_FRAME * readFrames + SCRATCH_SORT + SCRATCH_PER_WRITE_FRAME * writeFrames
122
+ estimate := f.size({ default: 0 }).times(factor)
123
+ return f.when(estimate.lt(f.gib(SCRATCH_DEVICE_MIN_GIB)), 0).otherwise(estimate)
124
+ }
125
+
79
126
  export {
80
127
  ramFormula: ramFormula,
81
128
  cpuFormula: cpuFormula,
129
+ scratchFormula: scratchFormula,
82
130
 
83
131
 
84
132
  RAM_INTERCEPT_GIB: RAM_INTERCEPT_GIB,
@@ -86,5 +134,9 @@ export {
86
134
  RAM_FLOOR_GIB: RAM_FLOOR_GIB,
87
135
  RAM_CAP_GIB: RAM_CAP_GIB,
88
136
  CPU_MIN: CPU_MIN,
89
- CPU_MAX: CPU_MAX
137
+ CPU_MAX: CPU_MAX,
138
+ SCRATCH_PER_READ_FRAME: SCRATCH_PER_READ_FRAME,
139
+ SCRATCH_SORT: SCRATCH_SORT,
140
+ SCRATCH_PER_WRITE_FRAME: SCRATCH_PER_WRITE_FRAME,
141
+ SCRATCH_DEVICE_MIN_GIB: SCRATCH_DEVICE_MIN_GIB
90
142
  }
Binary file
Binary file
Binary file
Binary file
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platforma-sdk/workflow-tengo",
3
- "version": "6.11.1",
3
+ "version": "6.12.0",
4
4
  "description": "Tengo SDK library",
5
5
  "type": "module",
6
6
  "types": "./dist/index.d.ts",
@@ -19,11 +19,11 @@
19
19
  "@milaboratories/pframes-rs-wasip2": "1.1.61",
20
20
  "@milaboratories/software-pframes-conv": "2.2.9",
21
21
  "@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
22
- "@platforma-open/milaboratories.software-ptabler": "2.1.13",
22
+ "@platforma-open/milaboratories.software-ptabler": "2.1.14",
23
23
  "@platforma-open/milaboratories.software-ptexter": "1.2.6"
24
24
  },
25
25
  "devDependencies": {
26
- "@platforma-sdk/tengo-builder": "4.1.1"
26
+ "@platforma-sdk/tengo-builder": "4.1.2"
27
27
  },
28
28
  "scripts": {
29
29
  "types:check": "pl-tengo imports && pl-tengo check",