@platforma-sdk/workflow-tengo 6.11.1 → 6.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/tengo/lib/exec.formula.lib.tengo +16 -0
  2. package/dist/tengo/lib/pt.lib.tengo +21 -0
  3. package/dist/tengo/lib/pt.sizing.lib.tengo +59 -1
  4. package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
  5. package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
  6. package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
  7. package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
  8. package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
  9. package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
  10. package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
  11. package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
  12. package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
  13. package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
  14. package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
  15. package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
  16. package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
  17. package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
  18. package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
  19. package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
  20. package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
  21. package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
  22. package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
  23. package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
  24. package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
  25. package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
  26. package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
  27. package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
  28. package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
  29. package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
  30. package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
  31. package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
  32. package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
  33. package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
  34. package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
  35. package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
  36. package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
  37. package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
  38. package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
  39. package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
  40. package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
  41. package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
  42. package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
  43. package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
  44. package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
  45. package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
  46. package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
  47. package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
  48. package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
  49. package/package.json +4 -4
@@ -210,6 +210,21 @@ evaluateResource := func(node, resolver, label) {
210
210
 
211
211
 
212
212
 
213
+ evaluateScratch := func(node, resolver) {
214
+ node = _unwrap(node)
215
+ ll.assert(!sets.hasElement(_BOOL_OPS, node.op),
216
+ "exec resource formula (scratchFreeSpace) must compute a number, but its top-level op is %q (a comparison/logic op). Use comparisons and logic only inside f.when(...).",
217
+ node.op)
218
+ v := evaluate(node, resolver)
219
+ ll.assert(is_int(v) && v >= 0,
220
+ "exec resource formula (scratchFreeSpace) must evaluate to a non-negative integer, got %v. Floor a subtraction that can underflow with .atLeast(0).",
221
+ v)
222
+ return v
223
+ }
224
+
225
+
226
+
227
+
213
228
 
214
229
 
215
230
 
@@ -450,6 +465,7 @@ export {
450
465
  isFormula: func(x) { return is_map(x) && (!is_undefined(x.__isFormula__) || !is_undefined(x.op)) },
451
466
 
452
467
  evaluateResource: evaluateResource,
468
+ evaluateScratch: evaluateScratch,
453
469
  resolveAst: resolveAst,
454
470
  referencedMetricFiles: referencedMetricFiles,
455
471
  applyFormulaFallback: applyFormulaFallback,
@@ -412,6 +412,7 @@ workflow := func() {
412
412
  cpu := undefined
413
413
  mem := undefined
414
414
  memFloor := undefined
415
+ scratchFreeSpace := undefined
415
416
  inputCache := undefined
416
417
  queue := execConstants.MEDIUM_QUEUE
417
418
 
@@ -931,6 +932,25 @@ workflow := func() {
931
932
 
932
933
 
933
934
 
935
+
936
+
937
+
938
+ scratchFreeSpace: func(amount) {
939
+ ll.assert(is_string(amount) || (is_int(amount) && (amount == 0 || amount >= 16 * units.MiB)),
940
+ "pt.workflow.scratchFreeSpace: expected 0, at least 16 MiB in bytes, or a size string, got %v", amount)
941
+ scratchFreeSpace = amount
942
+ return self
943
+ },
944
+
945
+
946
+
947
+
948
+
949
+
950
+
951
+
952
+
953
+
934
954
  cacheInputs: func(time) {
935
955
  ll.assert(
936
956
  is_int(time),
@@ -1031,6 +1051,7 @@ workflow := func() {
1031
1051
  cpu: cpu,
1032
1052
  mem: mem,
1033
1053
  memFloor: memFloor,
1054
+ scratchFreeSpace: scratchFreeSpace,
1034
1055
  inputCache: inputCache
1035
1056
  }
1036
1057
  ptResult := render.create(workflowRunTpl, runInputs, {
@@ -18,6 +18,31 @@
18
18
 
19
19
 
20
20
 
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
21
46
 
22
47
 
23
48
 
@@ -33,11 +58,19 @@ RAM_FLOOR_GIB := 2
33
58
  RAM_CAP_GIB := 256
34
59
  RAM_STATIC_FALLBACK := "4GiB"
35
60
 
61
+ SCRATCH_PER_READ_FRAME := 16
62
+ SCRATCH_SORT := 16
63
+ SCRATCH_PER_WRITE_FRAME := 2
64
+ SCRATCH_DEVICE_MIN_GIB := 16
65
+
36
66
  CPU_MIN := 2
37
67
  CPU_MAX := 8
38
68
  CPU_GIB_PER_CORE := 16
39
69
 
40
70
 
71
+ POLARS_MAX_THREADS_EXPR := format("{min(%d,max(%d,system.cpu))}", CPU_MAX, CPU_MIN)
72
+
73
+
41
74
 
42
75
 
43
76
 
@@ -76,9 +109,30 @@ cpuFormula := func() {
76
109
  staticFallback(CPU_MIN)
77
110
  }
78
111
 
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+ scratchFormula := func(readFrames, writeFrames) {
120
+ ll.assert(is_int(readFrames) && readFrames >= 0 && is_int(writeFrames) && writeFrames >= 0,
121
+ "pt.sizing: frame counts must be non-negative integers, got %v and %v", readFrames, writeFrames)
122
+ if readFrames == 0 && writeFrames == 0 {
123
+ return 0
124
+ }
125
+ factor := SCRATCH_PER_READ_FRAME * readFrames + SCRATCH_SORT + SCRATCH_PER_WRITE_FRAME * writeFrames
126
+ estimate := f.size({ default: 0 }).times(factor)
127
+ return f.when(estimate.lt(f.gib(SCRATCH_DEVICE_MIN_GIB)), 0).otherwise(estimate)
128
+ }
129
+
79
130
  export {
80
131
  ramFormula: ramFormula,
81
132
  cpuFormula: cpuFormula,
133
+ scratchFormula: scratchFormula,
134
+
135
+ POLARS_MAX_THREADS_EXPR: POLARS_MAX_THREADS_EXPR,
82
136
 
83
137
 
84
138
  RAM_INTERCEPT_GIB: RAM_INTERCEPT_GIB,
@@ -86,5 +140,9 @@ export {
86
140
  RAM_FLOOR_GIB: RAM_FLOOR_GIB,
87
141
  RAM_CAP_GIB: RAM_CAP_GIB,
88
142
  CPU_MIN: CPU_MIN,
89
- CPU_MAX: CPU_MAX
143
+ CPU_MAX: CPU_MAX,
144
+ SCRATCH_PER_READ_FRAME: SCRATCH_PER_READ_FRAME,
145
+ SCRATCH_SORT: SCRATCH_SORT,
146
+ SCRATCH_PER_WRITE_FRAME: SCRATCH_PER_WRITE_FRAME,
147
+ SCRATCH_DEVICE_MIN_GIB: SCRATCH_DEVICE_MIN_GIB
90
148
  }
Binary file
Binary file
Binary file
Binary file
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platforma-sdk/workflow-tengo",
3
- "version": "6.11.1",
3
+ "version": "6.12.1",
4
4
  "description": "Tengo SDK library",
5
5
  "type": "module",
6
6
  "types": "./dist/index.d.ts",
@@ -19,11 +19,11 @@
19
19
  "@milaboratories/pframes-rs-wasip2": "1.1.61",
20
20
  "@milaboratories/software-pframes-conv": "2.2.9",
21
21
  "@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
22
- "@platforma-open/milaboratories.software-ptabler": "2.1.13",
23
- "@platforma-open/milaboratories.software-ptexter": "1.2.6"
22
+ "@platforma-open/milaboratories.software-ptexter": "1.2.6",
23
+ "@platforma-open/milaboratories.software-ptabler": "2.1.14"
24
24
  },
25
25
  "devDependencies": {
26
- "@platforma-sdk/tengo-builder": "4.1.1"
26
+ "@platforma-sdk/tengo-builder": "4.1.2"
27
27
  },
28
28
  "scripts": {
29
29
  "types:check": "pl-tengo imports && pl-tengo check",