@platforma-sdk/workflow-tengo 6.13.0 → 6.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tengo/lib/pt.lib.tengo +107 -7
- package/dist/tengo/lib/pt.sizing.lib.tengo +147 -22
- package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
- package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
- package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
- package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
- package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
- package/package.json +3 -3
|
@@ -410,8 +410,14 @@ workflow := func() {
|
|
|
410
410
|
pframeCounter := 0
|
|
411
411
|
|
|
412
412
|
cpu := undefined
|
|
413
|
+
cpuFloor := undefined
|
|
414
|
+
cpuIntercept := undefined
|
|
415
|
+
cpuSlope := undefined
|
|
416
|
+
cpuCeiling := undefined
|
|
413
417
|
mem := undefined
|
|
414
418
|
memFloor := undefined
|
|
419
|
+
memSlope := undefined
|
|
420
|
+
memIntercept := undefined
|
|
415
421
|
polarsMaxThreads := undefined
|
|
416
422
|
scratchFreeSpace := undefined
|
|
417
423
|
inputCache := undefined
|
|
@@ -870,6 +876,17 @@ workflow := func() {
|
|
|
870
876
|
|
|
871
877
|
|
|
872
878
|
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
|
|
873
890
|
cpu: func(amount) {
|
|
874
891
|
cpu = amount
|
|
875
892
|
return self
|
|
@@ -882,6 +899,61 @@ workflow := func() {
|
|
|
882
899
|
|
|
883
900
|
|
|
884
901
|
|
|
902
|
+
cpuFloor: func(cores) {
|
|
903
|
+
sizing.assertCpuCores(cores, "cpuFloor")
|
|
904
|
+
cpuFloor = cores
|
|
905
|
+
return self
|
|
906
|
+
},
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
cpuIntercept: func(cores) {
|
|
916
|
+
sizing.assertCpuCores(cores, "cpuIntercept")
|
|
917
|
+
cpuIntercept = cores
|
|
918
|
+
return self
|
|
919
|
+
},
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
cpuSlope: func(slope) {
|
|
931
|
+
sizing.assertCpuSlope(slope)
|
|
932
|
+
cpuSlope = slope
|
|
933
|
+
return self
|
|
934
|
+
},
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
cpuCeiling: func(cores) {
|
|
945
|
+
sizing.assertCpuCores(cores, "cpuCeiling")
|
|
946
|
+
cpuCeiling = cores
|
|
947
|
+
return self
|
|
948
|
+
},
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
|
|
885
957
|
|
|
886
958
|
|
|
887
959
|
|
|
@@ -909,6 +981,11 @@ workflow := func() {
|
|
|
909
981
|
|
|
910
982
|
|
|
911
983
|
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
|
|
912
989
|
mem: func(amount) {
|
|
913
990
|
mem = amount
|
|
914
991
|
return self
|
|
@@ -927,6 +1004,11 @@ workflow := func() {
|
|
|
927
1004
|
|
|
928
1005
|
|
|
929
1006
|
|
|
1007
|
+
memFloor: func(amount) {
|
|
1008
|
+
sizing.assertRamFloor(amount)
|
|
1009
|
+
memFloor = amount
|
|
1010
|
+
return self
|
|
1011
|
+
},
|
|
930
1012
|
|
|
931
1013
|
|
|
932
1014
|
|
|
@@ -934,13 +1016,25 @@ workflow := func() {
|
|
|
934
1016
|
|
|
935
1017
|
|
|
936
1018
|
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
memSlope: func(slope) {
|
|
1023
|
+
sizing.assertRamSlope(slope)
|
|
1024
|
+
memSlope = slope
|
|
1025
|
+
return self
|
|
1026
|
+
},
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
memIntercept: func(amount) {
|
|
1036
|
+
sizing.assertRamIntercept(amount)
|
|
1037
|
+
memIntercept = amount
|
|
944
1038
|
return self
|
|
945
1039
|
},
|
|
946
1040
|
|
|
@@ -1070,8 +1164,14 @@ workflow := func() {
|
|
|
1070
1164
|
runMetaInputs := {
|
|
1071
1165
|
queue: queue,
|
|
1072
1166
|
cpu: cpu,
|
|
1167
|
+
cpuFloor: cpuFloor,
|
|
1168
|
+
cpuIntercept: cpuIntercept,
|
|
1169
|
+
cpuSlope: cpuSlope,
|
|
1170
|
+
cpuCeiling: cpuCeiling,
|
|
1073
1171
|
mem: mem,
|
|
1074
1172
|
memFloor: memFloor,
|
|
1173
|
+
memSlope: memSlope,
|
|
1174
|
+
memIntercept: memIntercept,
|
|
1075
1175
|
polarsMaxThreads: polarsMaxThreads,
|
|
1076
1176
|
scratchFreeSpace: scratchFreeSpace,
|
|
1077
1177
|
inputCache: inputCache
|
|
@@ -41,6 +41,14 @@
|
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
44
52
|
|
|
45
53
|
|
|
46
54
|
|
|
@@ -54,11 +62,11 @@ f := import("@platforma-sdk/workflow-tengo:exec.formula")
|
|
|
54
62
|
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
55
63
|
units := import("@platforma-sdk/workflow-tengo:units")
|
|
56
64
|
|
|
57
|
-
RAM_INTERCEPT_GIB :=
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
65
|
+
RAM_INTERCEPT_GIB := 4
|
|
66
|
+
RAM_INTERCEPT_MIN_GIB := 4
|
|
67
|
+
RAM_SLOPE := 10
|
|
68
|
+
RAM_SLOPE_MIN := 0
|
|
69
|
+
RAM_FLOOR_GIB := 4
|
|
62
70
|
|
|
63
71
|
SCRATCH_PER_READ_FRAME := 16
|
|
64
72
|
SCRATCH_SORT := 16
|
|
@@ -110,25 +118,56 @@ polarsMaxThreadsExpr := func(threads) {
|
|
|
110
118
|
|
|
111
119
|
|
|
112
120
|
|
|
121
|
+
assertRamSlope := func(slope) {
|
|
122
|
+
ll.assert(is_int(slope) && slope >= RAM_SLOPE_MIN,
|
|
123
|
+
"pt.sizing: memory slope must be an integer of at least %d, got %v", RAM_SLOPE_MIN, slope)
|
|
124
|
+
}
|
|
113
125
|
|
|
114
126
|
|
|
115
|
-
ramFormula := func(memFloor) {
|
|
116
|
-
floor := f.gib(RAM_FLOOR_GIB)
|
|
117
|
-
cap := f.gib(RAM_CAP_GIB)
|
|
118
|
-
fallback := RAM_STATIC_FALLBACK
|
|
119
127
|
|
|
120
|
-
if !is_undefined(memFloor) {
|
|
121
128
|
|
|
122
129
|
|
|
123
130
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
131
|
+
assertRamIntercept := func(intercept) {
|
|
132
|
+
ll.assert(is_int(intercept) && intercept >= RAM_INTERCEPT_MIN_GIB * units.GiB,
|
|
133
|
+
"pt.sizing: memory intercept must be an integer number of bytes of at least %d GiB, got %v", RAM_INTERCEPT_MIN_GIB, intercept)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
assertRamFloor := func(floor) {
|
|
142
|
+
ll.assert(is_int(floor) && floor > 0,
|
|
143
|
+
"pt.sizing: memory floor must be a positive integer number of bytes, got %v", floor)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
|
|
129
147
|
|
|
130
|
-
|
|
131
|
-
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
ramFormula := func(opts) {
|
|
156
|
+
ll.assert(is_map(opts), "pt.sizing.ramFormula: expected an options map, got %v", opts)
|
|
157
|
+
|
|
158
|
+
floor := is_undefined(opts.floor) ? RAM_FLOOR_GIB * units.GiB : opts.floor
|
|
159
|
+
slope := is_undefined(opts.slope) ? RAM_SLOPE : opts.slope
|
|
160
|
+
intercept := is_undefined(opts.intercept) ? RAM_INTERCEPT_GIB * units.GiB : opts.intercept
|
|
161
|
+
assertRamFloor(floor)
|
|
162
|
+
assertRamSlope(slope)
|
|
163
|
+
assertRamIntercept(intercept)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
fallback := intercept > floor ? intercept : floor
|
|
168
|
+
|
|
169
|
+
return f.size({ default: 0 }).times(slope).plus(f.const(intercept)).
|
|
170
|
+
atLeast(f.const(floor)).
|
|
132
171
|
staticFallback(fallback)
|
|
133
172
|
}
|
|
134
173
|
|
|
@@ -138,10 +177,90 @@ ramFormula := func(memFloor) {
|
|
|
138
177
|
|
|
139
178
|
|
|
140
179
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
180
|
+
assertCpuCores := func(cores, name) {
|
|
181
|
+
ll.assert(is_int(cores) && cores >= 1,
|
|
182
|
+
"pt.sizing: %s must be an integer of at least 1, got %v", name, cores)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
assertCpuSlope := func(slope) {
|
|
191
|
+
ll.assert((is_int(slope) || is_float(slope)) && slope >= 0,
|
|
192
|
+
"pt.sizing: cpuSlope must be a number of at least 0, got %v", slope)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
cpuSlopeHundredths := func(slope) {
|
|
204
|
+
assertCpuSlope(slope)
|
|
205
|
+
return int(float(slope) * 100 + 0.000001)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
cpuFormula := func(...args) {
|
|
226
|
+
ll.assert(len(args) <= 1, "pt.sizing.cpuFormula: expected at most one options map")
|
|
227
|
+
opts := len(args) == 1 && !is_undefined(args[0]) ? args[0] : {}
|
|
228
|
+
ll.assert(is_map(opts), "pt.sizing.cpuFormula: expected an options map, got %v", opts)
|
|
229
|
+
|
|
230
|
+
floor := is_undefined(opts.floor) ? CPU_MIN : opts.floor
|
|
231
|
+
intercept := is_undefined(opts.intercept) ? CPU_MIN : opts.intercept
|
|
232
|
+
assertCpuCores(floor, "cpuFloor")
|
|
233
|
+
assertCpuCores(intercept, "cpuIntercept")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
slopeTerm := f.size({ default: 0 }).dividedByFloor(f.gib(CPU_GIB_PER_CORE))
|
|
238
|
+
if !is_undefined(opts.slope) {
|
|
239
|
+
slopeTerm = f.size({ default: 0 }).times(cpuSlopeHundredths(opts.slope)).dividedByFloor(f.gib(100))
|
|
240
|
+
}
|
|
241
|
+
curve := slopeTerm.plus(f.const(intercept))
|
|
242
|
+
defaultCurve := is_undefined(opts.slope) && is_undefined(opts.intercept)
|
|
243
|
+
|
|
244
|
+
ceiling := undefined
|
|
245
|
+
if !is_undefined(opts.ceiling) {
|
|
246
|
+
assertCpuCores(opts.ceiling, "cpuCeiling")
|
|
247
|
+
ll.assert(opts.ceiling >= floor,
|
|
248
|
+
"pt.sizing: cpuCeiling (%d) must not be less than cpuFloor (%d)", opts.ceiling, floor)
|
|
249
|
+
ceiling = opts.ceiling
|
|
250
|
+
} else if defaultCurve {
|
|
251
|
+
ceiling = floor > CPU_MAX ? floor : CPU_MAX
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
fallback := intercept > floor ? intercept : floor
|
|
256
|
+
if !is_undefined(ceiling) && fallback > ceiling {
|
|
257
|
+
fallback = ceiling
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
if is_undefined(ceiling) {
|
|
261
|
+
return curve.atLeast(f.const(floor)).staticFallback(fallback)
|
|
262
|
+
}
|
|
263
|
+
return curve.between(f.const(floor), f.const(ceiling)).staticFallback(fallback)
|
|
145
264
|
}
|
|
146
265
|
|
|
147
266
|
|
|
@@ -164,7 +283,12 @@ scratchFormula := func(readFrames, writeFrames) {
|
|
|
164
283
|
|
|
165
284
|
export {
|
|
166
285
|
ramFormula: ramFormula,
|
|
286
|
+
assertRamSlope: assertRamSlope,
|
|
287
|
+
assertRamIntercept: assertRamIntercept,
|
|
288
|
+
assertRamFloor: assertRamFloor,
|
|
167
289
|
cpuFormula: cpuFormula,
|
|
290
|
+
assertCpuCores: assertCpuCores,
|
|
291
|
+
assertCpuSlope: assertCpuSlope,
|
|
168
292
|
scratchFormula: scratchFormula,
|
|
169
293
|
polarsMaxThreadsExpr: polarsMaxThreadsExpr,
|
|
170
294
|
assertPolarsMaxThreads: assertPolarsMaxThreads,
|
|
@@ -175,9 +299,10 @@ export {
|
|
|
175
299
|
|
|
176
300
|
|
|
177
301
|
RAM_INTERCEPT_GIB: RAM_INTERCEPT_GIB,
|
|
302
|
+
RAM_INTERCEPT_MIN_GIB: RAM_INTERCEPT_MIN_GIB,
|
|
178
303
|
RAM_SLOPE: RAM_SLOPE,
|
|
304
|
+
RAM_SLOPE_MIN: RAM_SLOPE_MIN,
|
|
179
305
|
RAM_FLOOR_GIB: RAM_FLOOR_GIB,
|
|
180
|
-
RAM_CAP_GIB: RAM_CAP_GIB,
|
|
181
306
|
CPU_MIN: CPU_MIN,
|
|
182
307
|
CPU_MAX: CPU_MAX,
|
|
183
308
|
SCRATCH_PER_READ_FRAME: SCRATCH_PER_READ_FRAME,
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platforma-sdk/workflow-tengo",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.15.0",
|
|
4
4
|
"description": "Tengo SDK library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
"@milaboratories/pframes-rs-wasip2": "1.1.61",
|
|
20
20
|
"@milaboratories/software-pframes-conv": "2.2.9",
|
|
21
21
|
"@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
|
|
22
|
-
"@platforma-open/milaboratories.software-
|
|
23
|
-
"@platforma-open/milaboratories.software-
|
|
22
|
+
"@platforma-open/milaboratories.software-ptabler": "2.1.14",
|
|
23
|
+
"@platforma-open/milaboratories.software-ptexter": "1.2.6"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@platforma-sdk/tengo-builder": "4.1.9"
|