@platforma-sdk/workflow-tengo 6.13.0 → 6.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/tengo/lib/pt.lib.tengo +107 -7
  2. package/dist/tengo/lib/pt.sizing.lib.tengo +147 -22
  3. package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
  4. package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
  5. package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
  6. package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
  7. package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
  8. package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
  9. package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
  10. package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
  11. package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
  12. package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
  13. package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
  14. package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
  15. package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
  16. package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
  17. package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
  18. package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
  19. package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
  20. package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
  21. package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
  22. package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
  23. package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
  24. package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
  25. package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
  26. package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
  27. package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
  28. package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
  29. package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
  30. package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
  31. package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
  32. package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
  33. package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
  34. package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
  35. package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
  36. package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
  37. package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
  38. package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
  39. package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
  40. package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
  41. package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
  42. package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
  43. package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
  44. package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
  45. package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
  46. package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
  47. package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
  48. package/package.json +3 -3
@@ -410,8 +410,14 @@ workflow := func() {
410
410
  pframeCounter := 0
411
411
 
412
412
  cpu := undefined
413
+ cpuFloor := undefined
414
+ cpuIntercept := undefined
415
+ cpuSlope := undefined
416
+ cpuCeiling := undefined
413
417
  mem := undefined
414
418
  memFloor := undefined
419
+ memSlope := undefined
420
+ memIntercept := undefined
415
421
  polarsMaxThreads := undefined
416
422
  scratchFreeSpace := undefined
417
423
  inputCache := undefined
@@ -870,6 +876,17 @@ workflow := func() {
870
876
 
871
877
 
872
878
 
879
+
880
+
881
+
882
+
883
+
884
+
885
+
886
+
887
+
888
+
889
+
873
890
  cpu: func(amount) {
874
891
  cpu = amount
875
892
  return self
@@ -882,6 +899,61 @@ workflow := func() {
882
899
 
883
900
 
884
901
 
902
+ cpuFloor: func(cores) {
903
+ sizing.assertCpuCores(cores, "cpuFloor")
904
+ cpuFloor = cores
905
+ return self
906
+ },
907
+
908
+
909
+
910
+
911
+
912
+
913
+
914
+
915
+ cpuIntercept: func(cores) {
916
+ sizing.assertCpuCores(cores, "cpuIntercept")
917
+ cpuIntercept = cores
918
+ return self
919
+ },
920
+
921
+
922
+
923
+
924
+
925
+
926
+
927
+
928
+
929
+
930
+ cpuSlope: func(slope) {
931
+ sizing.assertCpuSlope(slope)
932
+ cpuSlope = slope
933
+ return self
934
+ },
935
+
936
+
937
+
938
+
939
+
940
+
941
+
942
+
943
+
944
+ cpuCeiling: func(cores) {
945
+ sizing.assertCpuCores(cores, "cpuCeiling")
946
+ cpuCeiling = cores
947
+ return self
948
+ },
949
+
950
+
951
+
952
+
953
+
954
+
955
+
956
+
885
957
 
886
958
 
887
959
 
@@ -909,6 +981,11 @@ workflow := func() {
909
981
 
910
982
 
911
983
 
984
+
985
+
986
+
987
+
988
+
912
989
  mem: func(amount) {
913
990
  mem = amount
914
991
  return self
@@ -927,6 +1004,11 @@ workflow := func() {
927
1004
 
928
1005
 
929
1006
 
1007
+ memFloor: func(amount) {
1008
+ sizing.assertRamFloor(amount)
1009
+ memFloor = amount
1010
+ return self
1011
+ },
930
1012
 
931
1013
 
932
1014
 
@@ -934,13 +1016,25 @@ workflow := func() {
934
1016
 
935
1017
 
936
1018
 
937
- memFloor: func(amount) {
938
- ll.assert(is_int(amount) && amount > 0,
939
- "pt.workflow.memFloor: expected a positive number of bytes, got %v", amount)
940
- ll.assert(amount <= sizing.RAM_CAP_GIB * units.GiB,
941
- "pt.workflow.memFloor: floor of %v bytes exceeds the %d GiB ceiling of the auto-sized request. Use .mem() for a fixed request above the ceiling.",
942
- amount, sizing.RAM_CAP_GIB)
943
- memFloor = amount
1019
+
1020
+
1021
+
1022
+ memSlope: func(slope) {
1023
+ sizing.assertRamSlope(slope)
1024
+ memSlope = slope
1025
+ return self
1026
+ },
1027
+
1028
+
1029
+
1030
+
1031
+
1032
+
1033
+
1034
+
1035
+ memIntercept: func(amount) {
1036
+ sizing.assertRamIntercept(amount)
1037
+ memIntercept = amount
944
1038
  return self
945
1039
  },
946
1040
 
@@ -1070,8 +1164,14 @@ workflow := func() {
1070
1164
  runMetaInputs := {
1071
1165
  queue: queue,
1072
1166
  cpu: cpu,
1167
+ cpuFloor: cpuFloor,
1168
+ cpuIntercept: cpuIntercept,
1169
+ cpuSlope: cpuSlope,
1170
+ cpuCeiling: cpuCeiling,
1073
1171
  mem: mem,
1074
1172
  memFloor: memFloor,
1173
+ memSlope: memSlope,
1174
+ memIntercept: memIntercept,
1075
1175
  polarsMaxThreads: polarsMaxThreads,
1076
1176
  scratchFreeSpace: scratchFreeSpace,
1077
1177
  inputCache: inputCache
@@ -41,6 +41,14 @@
41
41
 
42
42
 
43
43
 
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
44
52
 
45
53
 
46
54
 
@@ -54,11 +62,11 @@ f := import("@platforma-sdk/workflow-tengo:exec.formula")
54
62
  ll := import("@platforma-sdk/workflow-tengo:ll")
55
63
  units := import("@platforma-sdk/workflow-tengo:units")
56
64
 
57
- RAM_INTERCEPT_GIB := 2
58
- RAM_SLOPE := 6
59
- RAM_FLOOR_GIB := 2
60
- RAM_CAP_GIB := 256
61
- RAM_STATIC_FALLBACK := "4GiB"
65
+ RAM_INTERCEPT_GIB := 4
66
+ RAM_INTERCEPT_MIN_GIB := 4
67
+ RAM_SLOPE := 10
68
+ RAM_SLOPE_MIN := 0
69
+ RAM_FLOOR_GIB := 4
62
70
 
63
71
  SCRATCH_PER_READ_FRAME := 16
64
72
  SCRATCH_SORT := 16
@@ -110,25 +118,56 @@ polarsMaxThreadsExpr := func(threads) {
110
118
 
111
119
 
112
120
 
121
+ assertRamSlope := func(slope) {
122
+ ll.assert(is_int(slope) && slope >= RAM_SLOPE_MIN,
123
+ "pt.sizing: memory slope must be an integer of at least %d, got %v", RAM_SLOPE_MIN, slope)
124
+ }
113
125
 
114
126
 
115
- ramFormula := func(memFloor) {
116
- floor := f.gib(RAM_FLOOR_GIB)
117
- cap := f.gib(RAM_CAP_GIB)
118
- fallback := RAM_STATIC_FALLBACK
119
127
 
120
- if !is_undefined(memFloor) {
121
128
 
122
129
 
123
130
 
124
- ll.assert(memFloor <= RAM_CAP_GIB * units.GiB,
125
- "pt.sizing: memFloor (%v bytes) must not exceed the %d GiB ceiling", memFloor, RAM_CAP_GIB)
126
- floor = f.const(memFloor)
127
- fallback = memFloor
128
- }
131
+ assertRamIntercept := func(intercept) {
132
+ ll.assert(is_int(intercept) && intercept >= RAM_INTERCEPT_MIN_GIB * units.GiB,
133
+ "pt.sizing: memory intercept must be an integer number of bytes of at least %d GiB, got %v", RAM_INTERCEPT_MIN_GIB, intercept)
134
+ }
135
+
136
+
137
+
138
+
139
+
140
+
141
+ assertRamFloor := func(floor) {
142
+ ll.assert(is_int(floor) && floor > 0,
143
+ "pt.sizing: memory floor must be a positive integer number of bytes, got %v", floor)
144
+ }
145
+
146
+
129
147
 
130
- return f.size({ default: 0 }).times(RAM_SLOPE).plus(f.gib(RAM_INTERCEPT_GIB)).
131
- between(floor, cap).
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+ ramFormula := func(opts) {
156
+ ll.assert(is_map(opts), "pt.sizing.ramFormula: expected an options map, got %v", opts)
157
+
158
+ floor := is_undefined(opts.floor) ? RAM_FLOOR_GIB * units.GiB : opts.floor
159
+ slope := is_undefined(opts.slope) ? RAM_SLOPE : opts.slope
160
+ intercept := is_undefined(opts.intercept) ? RAM_INTERCEPT_GIB * units.GiB : opts.intercept
161
+ assertRamFloor(floor)
162
+ assertRamSlope(slope)
163
+ assertRamIntercept(intercept)
164
+
165
+
166
+
167
+ fallback := intercept > floor ? intercept : floor
168
+
169
+ return f.size({ default: 0 }).times(slope).plus(f.const(intercept)).
170
+ atLeast(f.const(floor)).
132
171
  staticFallback(fallback)
133
172
  }
134
173
 
@@ -138,10 +177,90 @@ ramFormula := func(memFloor) {
138
177
 
139
178
 
140
179
 
141
- cpuFormula := func() {
142
- return f.size({ default: 0 }).dividedByFloor(f.gib(CPU_GIB_PER_CORE)).plus(f.const(CPU_MIN)).
143
- between(f.const(CPU_MIN), f.const(CPU_MAX)).
144
- staticFallback(CPU_MIN)
180
+ assertCpuCores := func(cores, name) {
181
+ ll.assert(is_int(cores) && cores >= 1,
182
+ "pt.sizing: %s must be an integer of at least 1, got %v", name, cores)
183
+ }
184
+
185
+
186
+
187
+
188
+
189
+
190
+ assertCpuSlope := func(slope) {
191
+ ll.assert((is_int(slope) || is_float(slope)) && slope >= 0,
192
+ "pt.sizing: cpuSlope must be a number of at least 0, got %v", slope)
193
+ }
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+
203
+ cpuSlopeHundredths := func(slope) {
204
+ assertCpuSlope(slope)
205
+ return int(float(slope) * 100 + 0.000001)
206
+ }
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+ cpuFormula := func(...args) {
226
+ ll.assert(len(args) <= 1, "pt.sizing.cpuFormula: expected at most one options map")
227
+ opts := len(args) == 1 && !is_undefined(args[0]) ? args[0] : {}
228
+ ll.assert(is_map(opts), "pt.sizing.cpuFormula: expected an options map, got %v", opts)
229
+
230
+ floor := is_undefined(opts.floor) ? CPU_MIN : opts.floor
231
+ intercept := is_undefined(opts.intercept) ? CPU_MIN : opts.intercept
232
+ assertCpuCores(floor, "cpuFloor")
233
+ assertCpuCores(intercept, "cpuIntercept")
234
+
235
+
236
+
237
+ slopeTerm := f.size({ default: 0 }).dividedByFloor(f.gib(CPU_GIB_PER_CORE))
238
+ if !is_undefined(opts.slope) {
239
+ slopeTerm = f.size({ default: 0 }).times(cpuSlopeHundredths(opts.slope)).dividedByFloor(f.gib(100))
240
+ }
241
+ curve := slopeTerm.plus(f.const(intercept))
242
+ defaultCurve := is_undefined(opts.slope) && is_undefined(opts.intercept)
243
+
244
+ ceiling := undefined
245
+ if !is_undefined(opts.ceiling) {
246
+ assertCpuCores(opts.ceiling, "cpuCeiling")
247
+ ll.assert(opts.ceiling >= floor,
248
+ "pt.sizing: cpuCeiling (%d) must not be less than cpuFloor (%d)", opts.ceiling, floor)
249
+ ceiling = opts.ceiling
250
+ } else if defaultCurve {
251
+ ceiling = floor > CPU_MAX ? floor : CPU_MAX
252
+ }
253
+
254
+
255
+ fallback := intercept > floor ? intercept : floor
256
+ if !is_undefined(ceiling) && fallback > ceiling {
257
+ fallback = ceiling
258
+ }
259
+
260
+ if is_undefined(ceiling) {
261
+ return curve.atLeast(f.const(floor)).staticFallback(fallback)
262
+ }
263
+ return curve.between(f.const(floor), f.const(ceiling)).staticFallback(fallback)
145
264
  }
146
265
 
147
266
 
@@ -164,7 +283,12 @@ scratchFormula := func(readFrames, writeFrames) {
164
283
 
165
284
  export {
166
285
  ramFormula: ramFormula,
286
+ assertRamSlope: assertRamSlope,
287
+ assertRamIntercept: assertRamIntercept,
288
+ assertRamFloor: assertRamFloor,
167
289
  cpuFormula: cpuFormula,
290
+ assertCpuCores: assertCpuCores,
291
+ assertCpuSlope: assertCpuSlope,
168
292
  scratchFormula: scratchFormula,
169
293
  polarsMaxThreadsExpr: polarsMaxThreadsExpr,
170
294
  assertPolarsMaxThreads: assertPolarsMaxThreads,
@@ -175,9 +299,10 @@ export {
175
299
 
176
300
 
177
301
  RAM_INTERCEPT_GIB: RAM_INTERCEPT_GIB,
302
+ RAM_INTERCEPT_MIN_GIB: RAM_INTERCEPT_MIN_GIB,
178
303
  RAM_SLOPE: RAM_SLOPE,
304
+ RAM_SLOPE_MIN: RAM_SLOPE_MIN,
179
305
  RAM_FLOOR_GIB: RAM_FLOOR_GIB,
180
- RAM_CAP_GIB: RAM_CAP_GIB,
181
306
  CPU_MIN: CPU_MIN,
182
307
  CPU_MAX: CPU_MAX,
183
308
  SCRATCH_PER_READ_FRAME: SCRATCH_PER_READ_FRAME,
Binary file
Binary file
Binary file
Binary file
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platforma-sdk/workflow-tengo",
3
- "version": "6.13.0",
3
+ "version": "6.15.0",
4
4
  "description": "Tengo SDK library",
5
5
  "type": "module",
6
6
  "types": "./dist/index.d.ts",
@@ -19,8 +19,8 @@
19
19
  "@milaboratories/pframes-rs-wasip2": "1.1.61",
20
20
  "@milaboratories/software-pframes-conv": "2.2.9",
21
21
  "@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
22
- "@platforma-open/milaboratories.software-ptexter": "1.2.6",
23
- "@platforma-open/milaboratories.software-ptabler": "2.1.14"
22
+ "@platforma-open/milaboratories.software-ptabler": "2.1.14",
23
+ "@platforma-open/milaboratories.software-ptexter": "1.2.6"
24
24
  },
25
25
  "devDependencies": {
26
26
  "@platforma-sdk/tengo-builder": "4.1.9"