@platforma-sdk/workflow-tengo 6.14.0 → 6.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tengo/lib/pframes.table-builder.lib.tengo +65 -4
- package/dist/tengo/lib/pt.lib.tengo +74 -0
- package/dist/tengo/lib/pt.sizing.lib.tengo +87 -4
- package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
- package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
- package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
- package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
- package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
- package/package.json +4 -4
|
@@ -46,6 +46,33 @@ VALID_FORMATS := {
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
VALID_COLUMN_JOINS := {
|
|
58
|
+
left: true,
|
|
59
|
+
full: true,
|
|
60
|
+
inner: true
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
columnJoinOpt := func(opts) {
|
|
65
|
+
if len(opts) == 0 || !ll.isMap(opts[0]) || is_undefined(opts[0].join) {
|
|
66
|
+
return undefined
|
|
67
|
+
}
|
|
68
|
+
join := opts[0].join
|
|
69
|
+
ll.assert(!is_undefined(VALID_COLUMN_JOINS[join]),
|
|
70
|
+
"join must be one of [left, full, inner], got '%v'", join)
|
|
71
|
+
return join
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
49
76
|
isUnresolved := func(value) {
|
|
50
77
|
stack := [value]
|
|
51
78
|
si := 0
|
|
@@ -142,6 +169,7 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
142
169
|
_primaries := []
|
|
143
170
|
_columns := []
|
|
144
171
|
_joinMode := "inner"
|
|
172
|
+
_columnJoin := "full"
|
|
145
173
|
_axisHeaders := []
|
|
146
174
|
_cpu := undefined
|
|
147
175
|
_mem := undefined
|
|
@@ -195,18 +223,23 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
195
223
|
|
|
196
224
|
|
|
197
225
|
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
|
|
198
229
|
addColumn: func(query, ...opts) {
|
|
199
230
|
header := undefined
|
|
200
231
|
if len(opts) > 0 && ll.isMap(opts[0]) {
|
|
201
232
|
header = opts[0].header
|
|
202
233
|
}
|
|
234
|
+
join := columnJoinOpt(opts)
|
|
203
235
|
if isUnresolved(query) || isEnrichmentRef(query) {
|
|
204
236
|
_hasUnresolved = true
|
|
205
237
|
}
|
|
206
238
|
_columns = append(_columns, {
|
|
207
239
|
query: query,
|
|
208
240
|
header: header,
|
|
209
|
-
mode: "single"
|
|
241
|
+
mode: "single",
|
|
242
|
+
join: join
|
|
210
243
|
})
|
|
211
244
|
return self
|
|
212
245
|
},
|
|
@@ -219,6 +252,9 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
219
252
|
|
|
220
253
|
|
|
221
254
|
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
|
|
222
258
|
addColumns: func(query, ...opts) {
|
|
223
259
|
headerPrefix := undefined
|
|
224
260
|
headerSuffix := undefined
|
|
@@ -226,6 +262,7 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
226
262
|
headerPrefix = opts[0].headerPrefix
|
|
227
263
|
headerSuffix = opts[0].headerSuffix
|
|
228
264
|
}
|
|
265
|
+
join := columnJoinOpt(opts)
|
|
229
266
|
|
|
230
267
|
|
|
231
268
|
|
|
@@ -254,7 +291,8 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
254
291
|
header: entryHeader,
|
|
255
292
|
headerPrefix: entryPrefix,
|
|
256
293
|
headerSuffix: entrySuffix,
|
|
257
|
-
mode: "multi"
|
|
294
|
+
mode: "multi",
|
|
295
|
+
join: join
|
|
258
296
|
})
|
|
259
297
|
}
|
|
260
298
|
return self
|
|
@@ -267,7 +305,8 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
267
305
|
query: query,
|
|
268
306
|
headerPrefix: headerPrefix,
|
|
269
307
|
headerSuffix: headerSuffix,
|
|
270
|
-
mode: "multi"
|
|
308
|
+
mode: "multi",
|
|
309
|
+
join: join
|
|
271
310
|
})
|
|
272
311
|
return self
|
|
273
312
|
},
|
|
@@ -291,6 +330,25 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
291
330
|
|
|
292
331
|
|
|
293
332
|
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
setColumnJoin: func(mode) {
|
|
340
|
+
ll.assert(!is_undefined(VALID_COLUMN_JOINS[mode]),
|
|
341
|
+
"mode must be one of [left, full, inner], got '%v'", mode)
|
|
342
|
+
_columnJoin = mode
|
|
343
|
+
return self
|
|
344
|
+
},
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
|
|
294
352
|
setAxisHeader: func(matcher, header) {
|
|
295
353
|
ll.assert(is_string(header), "header must be a string")
|
|
296
354
|
processedMatcher := undefined
|
|
@@ -469,7 +527,8 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
469
527
|
mode: c.mode,
|
|
470
528
|
header: c.header,
|
|
471
529
|
headerPrefix: c.headerPrefix,
|
|
472
|
-
headerSuffix: c.headerSuffix
|
|
530
|
+
headerSuffix: c.headerSuffix,
|
|
531
|
+
join: c.join
|
|
473
532
|
}
|
|
474
533
|
if isEnrichmentRef(c.query) {
|
|
475
534
|
resolvedHit := resolveColumn(c.query.hit)
|
|
@@ -560,12 +619,14 @@ tableBuilder := func(format, ...ctxArg) {
|
|
|
560
619
|
if !is_undefined(c.header) { entry.header = c.header }
|
|
561
620
|
if !is_undefined(c.headerPrefix) { entry.headerPrefix = c.headerPrefix }
|
|
562
621
|
if !is_undefined(c.headerSuffix) { entry.headerSuffix = c.headerSuffix }
|
|
622
|
+
if !is_undefined(c.join) { entry.join = c.join }
|
|
563
623
|
columnsMap[string(idx)] = entry
|
|
564
624
|
}
|
|
565
625
|
|
|
566
626
|
tplInputs := {
|
|
567
627
|
format: format,
|
|
568
628
|
joinMode: _joinMode,
|
|
629
|
+
columnJoin: _columnJoin,
|
|
569
630
|
axisHeaders: _axisHeaders,
|
|
570
631
|
primaryCount: len(primaryEntries),
|
|
571
632
|
columnCount: len(columnEntries),
|
|
@@ -410,6 +410,10 @@ workflow := func() {
|
|
|
410
410
|
pframeCounter := 0
|
|
411
411
|
|
|
412
412
|
cpu := undefined
|
|
413
|
+
cpuFloor := undefined
|
|
414
|
+
cpuIntercept := undefined
|
|
415
|
+
cpuSlope := undefined
|
|
416
|
+
cpuCeiling := undefined
|
|
413
417
|
mem := undefined
|
|
414
418
|
memFloor := undefined
|
|
415
419
|
memSlope := undefined
|
|
@@ -872,6 +876,17 @@ workflow := func() {
|
|
|
872
876
|
|
|
873
877
|
|
|
874
878
|
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
|
|
875
890
|
cpu: func(amount) {
|
|
876
891
|
cpu = amount
|
|
877
892
|
return self
|
|
@@ -884,6 +899,61 @@ workflow := func() {
|
|
|
884
899
|
|
|
885
900
|
|
|
886
901
|
|
|
902
|
+
cpuFloor: func(cores) {
|
|
903
|
+
sizing.assertCpuCores(cores, "cpuFloor")
|
|
904
|
+
cpuFloor = cores
|
|
905
|
+
return self
|
|
906
|
+
},
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
cpuIntercept: func(cores) {
|
|
916
|
+
sizing.assertCpuCores(cores, "cpuIntercept")
|
|
917
|
+
cpuIntercept = cores
|
|
918
|
+
return self
|
|
919
|
+
},
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
cpuSlope: func(slope) {
|
|
931
|
+
sizing.assertCpuSlope(slope)
|
|
932
|
+
cpuSlope = slope
|
|
933
|
+
return self
|
|
934
|
+
},
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
cpuCeiling: func(cores) {
|
|
945
|
+
sizing.assertCpuCores(cores, "cpuCeiling")
|
|
946
|
+
cpuCeiling = cores
|
|
947
|
+
return self
|
|
948
|
+
},
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
|
|
887
957
|
|
|
888
958
|
|
|
889
959
|
|
|
@@ -1094,6 +1164,10 @@ workflow := func() {
|
|
|
1094
1164
|
runMetaInputs := {
|
|
1095
1165
|
queue: queue,
|
|
1096
1166
|
cpu: cpu,
|
|
1167
|
+
cpuFloor: cpuFloor,
|
|
1168
|
+
cpuIntercept: cpuIntercept,
|
|
1169
|
+
cpuSlope: cpuSlope,
|
|
1170
|
+
cpuCeiling: cpuCeiling,
|
|
1097
1171
|
mem: mem,
|
|
1098
1172
|
memFloor: memFloor,
|
|
1099
1173
|
memSlope: memSlope,
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
|
|
58
|
+
|
|
58
59
|
|
|
59
60
|
|
|
60
61
|
f := import("@platforma-sdk/workflow-tengo:exec.formula")
|
|
@@ -176,10 +177,90 @@ ramFormula := func(opts) {
|
|
|
176
177
|
|
|
177
178
|
|
|
178
179
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
180
|
+
assertCpuCores := func(cores, name) {
|
|
181
|
+
ll.assert(is_int(cores) && cores >= 1,
|
|
182
|
+
"pt.sizing: %s must be an integer of at least 1, got %v", name, cores)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
assertCpuSlope := func(slope) {
|
|
191
|
+
ll.assert((is_int(slope) || is_float(slope)) && slope >= 0,
|
|
192
|
+
"pt.sizing: cpuSlope must be a number of at least 0, got %v", slope)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
cpuSlopeHundredths := func(slope) {
|
|
204
|
+
assertCpuSlope(slope)
|
|
205
|
+
return int(float(slope) * 100 + 0.000001)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
cpuFormula := func(...args) {
|
|
226
|
+
ll.assert(len(args) <= 1, "pt.sizing.cpuFormula: expected at most one options map")
|
|
227
|
+
opts := len(args) == 1 && !is_undefined(args[0]) ? args[0] : {}
|
|
228
|
+
ll.assert(is_map(opts), "pt.sizing.cpuFormula: expected an options map, got %v", opts)
|
|
229
|
+
|
|
230
|
+
floor := is_undefined(opts.floor) ? CPU_MIN : opts.floor
|
|
231
|
+
intercept := is_undefined(opts.intercept) ? CPU_MIN : opts.intercept
|
|
232
|
+
assertCpuCores(floor, "cpuFloor")
|
|
233
|
+
assertCpuCores(intercept, "cpuIntercept")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
slopeTerm := f.size({ default: 0 }).dividedByFloor(f.gib(CPU_GIB_PER_CORE))
|
|
238
|
+
if !is_undefined(opts.slope) {
|
|
239
|
+
slopeTerm = f.size({ default: 0 }).times(cpuSlopeHundredths(opts.slope)).dividedByFloor(f.gib(100))
|
|
240
|
+
}
|
|
241
|
+
curve := slopeTerm.plus(f.const(intercept))
|
|
242
|
+
defaultCurve := is_undefined(opts.slope) && is_undefined(opts.intercept)
|
|
243
|
+
|
|
244
|
+
ceiling := undefined
|
|
245
|
+
if !is_undefined(opts.ceiling) {
|
|
246
|
+
assertCpuCores(opts.ceiling, "cpuCeiling")
|
|
247
|
+
ll.assert(opts.ceiling >= floor,
|
|
248
|
+
"pt.sizing: cpuCeiling (%d) must not be less than cpuFloor (%d)", opts.ceiling, floor)
|
|
249
|
+
ceiling = opts.ceiling
|
|
250
|
+
} else if defaultCurve {
|
|
251
|
+
ceiling = floor > CPU_MAX ? floor : CPU_MAX
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
fallback := intercept > floor ? intercept : floor
|
|
256
|
+
if !is_undefined(ceiling) && fallback > ceiling {
|
|
257
|
+
fallback = ceiling
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
if is_undefined(ceiling) {
|
|
261
|
+
return curve.atLeast(f.const(floor)).staticFallback(fallback)
|
|
262
|
+
}
|
|
263
|
+
return curve.between(f.const(floor), f.const(ceiling)).staticFallback(fallback)
|
|
183
264
|
}
|
|
184
265
|
|
|
185
266
|
|
|
@@ -206,6 +287,8 @@ export {
|
|
|
206
287
|
assertRamIntercept: assertRamIntercept,
|
|
207
288
|
assertRamFloor: assertRamFloor,
|
|
208
289
|
cpuFormula: cpuFormula,
|
|
290
|
+
assertCpuCores: assertCpuCores,
|
|
291
|
+
assertCpuSlope: assertCpuSlope,
|
|
209
292
|
scratchFormula: scratchFormula,
|
|
210
293
|
polarsMaxThreadsExpr: polarsMaxThreadsExpr,
|
|
211
294
|
assertPolarsMaxThreads: assertPolarsMaxThreads,
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platforma-sdk/workflow-tengo",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.16.0",
|
|
4
4
|
"description": "Tengo SDK library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -19,11 +19,11 @@
|
|
|
19
19
|
"@milaboratories/pframes-rs-wasip2": "1.1.61",
|
|
20
20
|
"@milaboratories/software-pframes-conv": "2.2.9",
|
|
21
21
|
"@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
|
|
22
|
-
"@platforma-open/milaboratories.software-
|
|
23
|
-
"@platforma-open/milaboratories.software-
|
|
22
|
+
"@platforma-open/milaboratories.software-ptexter": "1.2.6",
|
|
23
|
+
"@platforma-open/milaboratories.software-ptabler": "2.1.14"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"@platforma-sdk/tengo-builder": "4.1.
|
|
26
|
+
"@platforma-sdk/tengo-builder": "4.1.11"
|
|
27
27
|
},
|
|
28
28
|
"scripts": {
|
|
29
29
|
"types:check": "pl-tengo imports && pl-tengo check",
|