@platforma-sdk/workflow-tengo 6.6.4 → 6.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tengo/lib/exec.constants.lib.tengo +2 -0
- package/dist/tengo/lib/exec.formula.lib.tengo +160 -44
- package/dist/tengo/lib/exec.lib.tengo +140 -29
- package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
- package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
- package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
- package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
- package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
- package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
- package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
- package/package.json +4 -4
|
@@ -27,6 +27,7 @@ RUNNER_EXECUTOR := "executor" // local binary execution
|
|
|
27
27
|
|
|
28
28
|
META_INPUT_MEM_FORMULA := "__memFormula__"
|
|
29
29
|
META_INPUT_CPU_FORMULA := "__cpuFormula__"
|
|
30
|
+
META_INPUT_GPU_FORMULA := "__gpuFormula__"
|
|
30
31
|
REGULAR_INPUT_LINE_COUNT := "__metric_lineCount__"
|
|
31
32
|
|
|
32
33
|
export ll.toStrict({
|
|
@@ -47,5 +48,6 @@ export ll.toStrict({
|
|
|
47
48
|
|
|
48
49
|
META_INPUT_MEM_FORMULA: META_INPUT_MEM_FORMULA,
|
|
49
50
|
META_INPUT_CPU_FORMULA: META_INPUT_CPU_FORMULA,
|
|
51
|
+
META_INPUT_GPU_FORMULA: META_INPUT_GPU_FORMULA,
|
|
50
52
|
REGULAR_INPUT_LINE_COUNT: REGULAR_INPUT_LINE_COUNT
|
|
51
53
|
})
|
|
@@ -18,6 +18,20 @@
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
21
35
|
|
|
22
36
|
|
|
23
37
|
|
|
@@ -51,19 +65,30 @@ units := import("@platforma-sdk/workflow-tengo:units")
|
|
|
51
65
|
|
|
52
66
|
|
|
53
67
|
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
54
71
|
_KNOWN_OPS := sets.fromSlice([
|
|
55
72
|
"const", "metric", "metricSum",
|
|
56
|
-
"
|
|
57
|
-
"
|
|
73
|
+
"plus", "minus", "times", "dividedBy", "dividedByFloor",
|
|
74
|
+
"atLeast", "atMost", "between",
|
|
58
75
|
"gt", "gte", "lt", "lte", "and", "or", "not",
|
|
59
|
-
"
|
|
76
|
+
"when"
|
|
60
77
|
])
|
|
61
78
|
|
|
62
79
|
|
|
63
80
|
|
|
64
81
|
_BOOL_OPS := sets.fromSlice(["gt", "gte", "lt", "lte", "and", "or", "not"])
|
|
65
82
|
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
_unwrap := func(x) {
|
|
87
|
+
return is_map(x) && !is_undefined(x.__isFormula__) ? x.__ast__ : x
|
|
88
|
+
}
|
|
89
|
+
|
|
66
90
|
_wrap := func(x) {
|
|
91
|
+
x = _unwrap(x)
|
|
67
92
|
if is_map(x) && !is_undefined(x.op) {
|
|
68
93
|
ll.assert(sets.hasElement(_KNOWN_OPS, x.op), "exec.formula: not a valid expression node (unknown op %v)", x.op)
|
|
69
94
|
return x
|
|
@@ -92,8 +117,22 @@ _metric := func(metricName, args) {
|
|
|
92
117
|
return node
|
|
93
118
|
}
|
|
94
119
|
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
_staticValue := func(v) {
|
|
124
|
+
if is_map(v) && !is_undefined(v.__isFormula__) {
|
|
125
|
+
a := v.__ast__
|
|
126
|
+
ll.assert(is_map(a) && a.op == "const",
|
|
127
|
+
"exec.formula.staticFallback: expected a constant (a number, a size string, or a pure unit like f.gib(64)), got a formula")
|
|
128
|
+
return a.value
|
|
129
|
+
}
|
|
130
|
+
return v
|
|
131
|
+
}
|
|
132
|
+
|
|
95
133
|
evaluate := undefined
|
|
96
134
|
evaluate = func(node, resolver) {
|
|
135
|
+
node = _unwrap(node)
|
|
97
136
|
op := node.op
|
|
98
137
|
if op == "const" { return node.value }
|
|
99
138
|
if op == "metricSum" {
|
|
@@ -102,15 +141,17 @@ evaluate = func(node, resolver) {
|
|
|
102
141
|
return fn(node.files)
|
|
103
142
|
}
|
|
104
143
|
|
|
105
|
-
if op == "
|
|
106
|
-
if op == "
|
|
107
|
-
if op == "
|
|
108
|
-
if op == "
|
|
144
|
+
if op == "plus" { return evaluate(node.a, resolver) + evaluate(node.b, resolver) }
|
|
145
|
+
if op == "minus" { return evaluate(node.a, resolver) - evaluate(node.b, resolver) }
|
|
146
|
+
if op == "times" { return evaluate(node.a, resolver) * evaluate(node.b, resolver) }
|
|
147
|
+
if op == "dividedByFloor" {
|
|
109
148
|
b := evaluate(node.b, resolver)
|
|
110
149
|
ll.assert(b != 0, "exec.formula.evaluate: division by zero")
|
|
150
|
+
|
|
151
|
+
|
|
111
152
|
return evaluate(node.a, resolver) / b
|
|
112
153
|
}
|
|
113
|
-
if op == "
|
|
154
|
+
if op == "dividedBy" {
|
|
114
155
|
b := evaluate(node.b, resolver)
|
|
115
156
|
ll.assert(b != 0, "exec.formula.evaluate: division by zero")
|
|
116
157
|
a := evaluate(node.a, resolver)
|
|
@@ -118,13 +159,13 @@ evaluate = func(node, resolver) {
|
|
|
118
159
|
return (a + b - 1) / b
|
|
119
160
|
}
|
|
120
161
|
|
|
121
|
-
if op == "
|
|
122
|
-
if op == "
|
|
123
|
-
if op == "
|
|
162
|
+
if op == "atLeast" { a := evaluate(node.a, resolver); b := evaluate(node.b, resolver); return a > b ? a : b }
|
|
163
|
+
if op == "atMost" { a := evaluate(node.a, resolver); b := evaluate(node.b, resolver); return a < b ? a : b }
|
|
164
|
+
if op == "between" {
|
|
124
165
|
v := evaluate(node.value, resolver)
|
|
125
166
|
lo := evaluate(node.lo, resolver)
|
|
126
167
|
hi := evaluate(node.hi, resolver)
|
|
127
|
-
ll.assert(lo <= hi, "exec.formula.
|
|
168
|
+
ll.assert(lo <= hi, "exec.formula.between: lo (%v) must be <= hi (%v) — inverted bounds collapse every input to hi", lo, hi)
|
|
128
169
|
if v < lo { v = lo }
|
|
129
170
|
if v > hi { v = hi }
|
|
130
171
|
return v
|
|
@@ -138,7 +179,7 @@ evaluate = func(node, resolver) {
|
|
|
138
179
|
if op == "or" { return evaluate(node.a, resolver) || evaluate(node.b, resolver) }
|
|
139
180
|
if op == "not" { return !evaluate(node.a, resolver) }
|
|
140
181
|
|
|
141
|
-
if op == "
|
|
182
|
+
if op == "when" { return evaluate(node.cond, resolver) ? evaluate(node.then, resolver) : evaluate(node["else"], resolver) }
|
|
142
183
|
|
|
143
184
|
ll.panic("exec.formula.evaluate: unknown op %q", op)
|
|
144
185
|
}
|
|
@@ -153,12 +194,13 @@ evaluate = func(node, resolver) {
|
|
|
153
194
|
|
|
154
195
|
|
|
155
196
|
evaluateResource := func(node, resolver, label) {
|
|
197
|
+
node = _unwrap(node)
|
|
156
198
|
ll.assert(!sets.hasElement(_BOOL_OPS, node.op),
|
|
157
|
-
"exec resource formula (%s) must compute a number, but its top-level op is %q (a comparison/logic op). Use comparisons and logic only inside f.
|
|
199
|
+
"exec resource formula (%s) must compute a number, but its top-level op is %q (a comparison/logic op). Use comparisons and logic only inside f.when(...).",
|
|
158
200
|
label, node.op)
|
|
159
201
|
v := evaluate(node, resolver)
|
|
160
202
|
ll.assert(is_int(v) && v > 0,
|
|
161
|
-
"exec resource formula (%s) must evaluate to a positive integer, got %v. If a metric can be zero or a subtraction can underflow, floor it with
|
|
203
|
+
"exec resource formula (%s) must evaluate to a positive integer, got %v. If a metric can be zero or a subtraction can underflow, floor it with .atLeast(...) or .between(...).",
|
|
162
204
|
label, v)
|
|
163
205
|
return v
|
|
164
206
|
}
|
|
@@ -192,6 +234,7 @@ _assertMetricCompatible := func(metric, fileName) {
|
|
|
192
234
|
|
|
193
235
|
resolveAst := undefined
|
|
194
236
|
resolveAst = func(node, tagMap, allFiles) {
|
|
237
|
+
node = _unwrap(node)
|
|
195
238
|
op := node.op
|
|
196
239
|
if op == "metric" {
|
|
197
240
|
files := is_undefined(node.tag) ? allFiles : (is_undefined(tagMap[node.tag]) ? [] : tagMap[node.tag])
|
|
@@ -221,6 +264,7 @@ resolveAst = func(node, tagMap, allFiles) {
|
|
|
221
264
|
}
|
|
222
265
|
|
|
223
266
|
referencedMetricFiles := func(resolvedNode) {
|
|
267
|
+
resolvedNode = _unwrap(resolvedNode)
|
|
224
268
|
acc := {}
|
|
225
269
|
walk := undefined
|
|
226
270
|
walk = func(n) {
|
|
@@ -255,63 +299,135 @@ referencedMetricFiles := func(resolvedNode) {
|
|
|
255
299
|
|
|
256
300
|
|
|
257
301
|
|
|
302
|
+
|
|
303
|
+
|
|
258
304
|
applyFormulaFallback := func(dims) {
|
|
259
|
-
|
|
260
|
-
|
|
305
|
+
gpu := is_undefined(dims.gpu) ? {} : dims.gpu
|
|
306
|
+
res := { ram: dims.ram.static, cpu: dims.cpu.static, gpu: gpu.static, err: undefined }
|
|
307
|
+
if !is_undefined(_unwrap(dims.ram.ast)) && is_undefined(res.ram) {
|
|
261
308
|
if is_undefined(dims.ram.fallback) {
|
|
262
|
-
res.err = "exec.builder:
|
|
309
|
+
res.err = "exec.builder: the RAM formula needs a static fallback on backends that can't evaluate resource formulas at run time. Provide .staticFallback(...) on the RAM formula, set a static ram, or upgrade the backend."
|
|
263
310
|
return res
|
|
264
311
|
}
|
|
265
312
|
res.ram = dims.ram.fallback
|
|
266
313
|
}
|
|
267
|
-
if !is_undefined(dims.cpu.ast) && is_undefined(res.cpu) {
|
|
314
|
+
if !is_undefined(_unwrap(dims.cpu.ast)) && is_undefined(res.cpu) {
|
|
268
315
|
if is_undefined(dims.cpu.fallback) {
|
|
269
|
-
res.err = "exec.builder:
|
|
316
|
+
res.err = "exec.builder: the CPU formula needs a static fallback on backends that can't evaluate resource formulas at run time. Provide .staticFallback(...) on the CPU formula, set a static cpu, or upgrade the backend."
|
|
270
317
|
return res
|
|
271
318
|
}
|
|
272
319
|
res.cpu = dims.cpu.fallback
|
|
273
320
|
}
|
|
321
|
+
if !is_undefined(_unwrap(gpu.ast)) && is_undefined(res.gpu) {
|
|
322
|
+
if is_undefined(gpu.fallback) {
|
|
323
|
+
res.err = "exec.builder: the VRAM formula needs a static fallback on backends that can't evaluate resource formulas at run time. Provide .staticFallback(...) on the vram formula, set a static vram, or upgrade the backend."
|
|
324
|
+
return res
|
|
325
|
+
}
|
|
326
|
+
res.gpu = gpu.fallback
|
|
327
|
+
}
|
|
274
328
|
return res
|
|
275
329
|
}
|
|
276
330
|
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
_node := undefined
|
|
342
|
+
_node = func(ast, fallback) {
|
|
343
|
+
self := {
|
|
344
|
+
__isFormula__: true,
|
|
345
|
+
__ast__: ast,
|
|
346
|
+
__fallback__: fallback
|
|
347
|
+
}
|
|
348
|
+
self.plus = func(x) { return _node(_bin("plus", ast, x), fallback) }
|
|
349
|
+
self.minus = func(x) { return _node(_bin("minus", ast, x), fallback) }
|
|
350
|
+
self.times = func(x) { return _node(_bin("times", ast, x), fallback) }
|
|
351
|
+
self.dividedBy = func(x) { return _node(_bin("dividedBy", ast, x), fallback) }
|
|
352
|
+
self.dividedByFloor = func(x) { return _node(_bin("dividedByFloor", ast, x), fallback) }
|
|
353
|
+
self.atLeast = func(x) { return _node(_bin("atLeast", ast, x), fallback) }
|
|
354
|
+
self.atMost = func(x) { return _node(_bin("atMost", ast, x), fallback) }
|
|
355
|
+
self.between = func(lo, hi) { return _node({ op: "between", value: _wrap(ast), lo: _wrap(lo), hi: _wrap(hi) }, fallback) }
|
|
356
|
+
self.gt = func(x) { return _node(_bin("gt", ast, x), fallback) }
|
|
357
|
+
self.gte = func(x) { return _node(_bin("gte", ast, x), fallback) }
|
|
358
|
+
self.lt = func(x) { return _node(_bin("lt", ast, x), fallback) }
|
|
359
|
+
self.lte = func(x) { return _node(_bin("lte", ast, x), fallback) }
|
|
360
|
+
self.and = func(x) { return _node(_bin("and", ast, x), fallback) }
|
|
361
|
+
self.or = func(x) { return _node(_bin("or", ast, x), fallback) }
|
|
362
|
+
self.not = func() { return _node({ op: "not", a: _wrap(ast) }, fallback) }
|
|
363
|
+
self.staticFallback = func(v) { return _node(ast, _staticValue(v)) }
|
|
364
|
+
return self
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
_condBuilder := undefined
|
|
372
|
+
_condBuilder = func(branches) {
|
|
373
|
+
self := {}
|
|
374
|
+
self.when = func(cond, value) {
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
next := []
|
|
378
|
+
for br in branches { next = append(next, br) }
|
|
379
|
+
next = append(next, { cond: _wrap(cond), value: _wrap(value) })
|
|
380
|
+
return _condBuilder(next)
|
|
381
|
+
}
|
|
382
|
+
self.otherwise = func(dflt) {
|
|
383
|
+
node := _wrap(dflt)
|
|
384
|
+
i := len(branches) - 1
|
|
385
|
+
for i >= 0 {
|
|
386
|
+
b := branches[i]
|
|
387
|
+
node = { op: "when", cond: b.cond, then: b.value, "else": node }
|
|
388
|
+
i = i - 1
|
|
389
|
+
}
|
|
390
|
+
return _node(node, undefined)
|
|
391
|
+
}
|
|
392
|
+
return self
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
_const := func(n) { ll.assert(is_int(n), "exec.formula: unit amount must be an integer"); return _node({ op: "const", value: n }, undefined) }
|
|
396
|
+
|
|
277
397
|
export {
|
|
278
398
|
|
|
279
399
|
|
|
280
400
|
|
|
281
401
|
|
|
282
402
|
|
|
283
|
-
size: func(...args) { return _metric("size", args) },
|
|
284
|
-
lineCount: func(...args) { return _metric("lineCount", args) },
|
|
403
|
+
size: func(...args) { return _node(_metric("size", args), undefined) },
|
|
404
|
+
lineCount: func(...args) { return _node(_metric("lineCount", args), undefined) },
|
|
405
|
+
|
|
406
|
+
gib: func(n) { return _const(n * units.GiB) },
|
|
407
|
+
mib: func(n) { return _const(n * units.MiB) },
|
|
408
|
+
kib: func(n) { return _const(n * units.KiB) },
|
|
409
|
+
gb: func(n) { return _const(n * units.GB) },
|
|
410
|
+
mb: func(n) { return _const(n * units.MB) },
|
|
411
|
+
kb: func(n) { return _const(n * units.KB) },
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
const: func(n) { return _const(n) },
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
|
|
285
419
|
|
|
286
|
-
gib: func(n) { ll.assert(is_int(n), "exec.formula.gib: n must be an integer"); return { op: "const", value: n * units.GiB } },
|
|
287
|
-
mib: func(n) { ll.assert(is_int(n), "exec.formula.mib: n must be an integer"); return { op: "const", value: n * units.MiB } },
|
|
288
|
-
kib: func(n) { ll.assert(is_int(n), "exec.formula.kib: n must be an integer"); return { op: "const", value: n * units.KiB } },
|
|
289
|
-
gb: func(n) { ll.assert(is_int(n), "exec.formula.gb: n must be an integer"); return { op: "const", value: n * units.GB } },
|
|
290
|
-
mb: func(n) { ll.assert(is_int(n), "exec.formula.mb: n must be an integer"); return { op: "const", value: n * units.MB } },
|
|
291
|
-
kb: func(n) { ll.assert(is_int(n), "exec.formula.kb: n must be an integer"); return { op: "const", value: n * units.KB } },
|
|
292
420
|
|
|
293
|
-
|
|
294
|
-
sub: func(a, b) { return _bin("sub", a, b) },
|
|
295
|
-
mul: func(a, b) { return _bin("mul", a, b) },
|
|
296
|
-
div: func(a, b) { return _bin("div", a, b) },
|
|
421
|
+
when: func(cond, value) { return _condBuilder([]).when(cond, value) },
|
|
297
422
|
|
|
298
423
|
|
|
299
424
|
|
|
300
|
-
divCeil: func(a, b) { return _bin("divCeil", a, b) },
|
|
301
425
|
|
|
302
|
-
max: func(a, b) { return _bin("max", a, b) },
|
|
303
|
-
min: func(a, b) { return _bin("min", a, b) },
|
|
304
|
-
clamp: func(value, lo, hi) { return { op: "clamp", value: _wrap(value), lo: _wrap(lo), hi: _wrap(hi) } },
|
|
305
426
|
|
|
306
|
-
gt: func(a, b) { return _bin("gt", a, b) },
|
|
307
|
-
gte: func(a, b) { return _bin("gte", a, b) },
|
|
308
|
-
lt: func(a, b) { return _bin("lt", a, b) },
|
|
309
|
-
lte: func(a, b) { return _bin("lte", a, b) },
|
|
310
|
-
and: func(a, b) { return _bin("and", a, b) },
|
|
311
|
-
or: func(a, b) { return _bin("or", a, b) },
|
|
312
|
-
not: func(a) { return { op: "not", a: _wrap(a) } },
|
|
313
427
|
|
|
314
|
-
|
|
428
|
+
astOf: func(x) { return _unwrap(x) },
|
|
429
|
+
fallbackOf: func(x) { return is_map(x) && !is_undefined(x.__isFormula__) ? x.__fallback__ : undefined },
|
|
430
|
+
isFormula: func(x) { return is_map(x) && (!is_undefined(x.__isFormula__) || !is_undefined(x.op)) },
|
|
315
431
|
|
|
316
432
|
evaluateResource: evaluateResource,
|
|
317
433
|
resolveAst: resolveAst,
|
|
@@ -42,6 +42,7 @@ _lineCountResource := undefined
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
|
|
45
|
+
|
|
45
46
|
_assertValidCpuRequest := func(amount, ctx) {
|
|
46
47
|
validation.assertType(amount, "number", ctx + ": amount of cores must be a number")
|
|
47
48
|
ll.assert(amount > 0, ctx + ": amount of cores should be greater than 0")
|
|
@@ -54,6 +55,14 @@ _assertValidMemRequest := func(amount, ctx) {
|
|
|
54
55
|
|
|
55
56
|
|
|
56
57
|
|
|
58
|
+
_assertValidGpuRequest := func(amount, ctx) {
|
|
59
|
+
validation.assertType(amount, ["or", "number", "string"], ctx + ": VRAM amount must be a number of bytes or a size string like '16GiB'")
|
|
60
|
+
ll.assert(is_string(amount) || amount >= 16 * units.MiB, ctx + ": VRAM amount in bytes should be at least 16MiB (did you pass bytes instead of GiB?)")
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
57
66
|
|
|
58
67
|
|
|
59
68
|
builder := func() {
|
|
@@ -85,10 +94,15 @@ builder := func() {
|
|
|
85
94
|
gpuMemory := undefined
|
|
86
95
|
|
|
87
96
|
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
|
|
88
100
|
memFormulaAst := undefined
|
|
89
101
|
memFormulaFallback := undefined
|
|
90
102
|
cpuFormulaAst := undefined
|
|
91
103
|
cpuFormulaFallback := undefined
|
|
104
|
+
gpuFormulaAst := undefined
|
|
105
|
+
gpuFormulaFallback := undefined
|
|
92
106
|
fileTags := {} // fileName -> tag (for f.size("tag") / f.lineCount("tag") aggregation)
|
|
93
107
|
|
|
94
108
|
commandName := ""
|
|
@@ -503,20 +517,6 @@ builder := func() {
|
|
|
503
517
|
|
|
504
518
|
|
|
505
519
|
|
|
506
|
-
memFormula: func(ast, ...opts) {
|
|
507
|
-
ll.assert(is_map(ast) && !is_undefined(ast.op), "exec.builder.memFormula: expected a formula node from exec.formula")
|
|
508
|
-
memFormulaAst = ast
|
|
509
|
-
if len(opts) > 0 && ll.isMap(opts[0]) {
|
|
510
|
-
for k, _ in opts[0] {
|
|
511
|
-
ll.assert(k == "fallback", "exec.builder.memFormula: unknown option %q (only { fallback } is supported)", k)
|
|
512
|
-
}
|
|
513
|
-
if !is_undefined(opts[0].fallback) {
|
|
514
|
-
_assertValidMemRequest(opts[0].fallback, "exec.builder.memFormula fallback")
|
|
515
|
-
memFormulaFallback = opts[0].fallback
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
return self
|
|
519
|
-
},
|
|
520
520
|
|
|
521
521
|
|
|
522
522
|
|
|
@@ -541,21 +541,116 @@ builder := func() {
|
|
|
541
541
|
|
|
542
542
|
|
|
543
543
|
|
|
544
|
+
resources: func(spec) {
|
|
545
|
+
ll.assert(ll.isMap(spec), "exec.builder.resources: expected a config map")
|
|
546
|
+
allowed := { queue: true, onCPU: true, onGPU: true }
|
|
547
|
+
for k, _ in spec {
|
|
548
|
+
ll.assert(!is_undefined(allowed[k]), "exec.builder.resources: unknown key %q (allowed: queue, onCPU, onGPU)", k)
|
|
549
|
+
}
|
|
550
|
+
hasCPU := !is_undefined(spec.onCPU)
|
|
551
|
+
hasGPU := !is_undefined(spec.onGPU)
|
|
552
|
+
ll.assert(hasCPU || hasGPU, "exec.builder.resources: provide onCPU and/or onGPU")
|
|
553
|
+
|
|
554
|
+
if !is_undefined(spec.queue) {
|
|
555
|
+
validation.assertType(spec.queue, "string", "exec.builder.resources: queue must be a string")
|
|
556
|
+
queue = spec.queue
|
|
557
|
+
}
|
|
544
558
|
|
|
545
559
|
|
|
546
560
|
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
setCpu := func(v, ctx) {
|
|
568
|
+
if execFormula.isFormula(v) {
|
|
569
|
+
cpuFormulaAst = execFormula.astOf(v)
|
|
570
|
+
cpuFormulaFallback = undefined
|
|
571
|
+
fb := execFormula.fallbackOf(v)
|
|
572
|
+
if !is_undefined(fb) { _assertValidCpuRequest(fb, ctx + " cpu fallback"); cpuFormulaFallback = fb }
|
|
573
|
+
cpuRequest = undefined
|
|
574
|
+
} else {
|
|
575
|
+
_assertValidCpuRequest(v, ctx + " cpu")
|
|
576
|
+
cpuRequest = v
|
|
577
|
+
cpuFormulaAst = undefined
|
|
578
|
+
cpuFormulaFallback = undefined
|
|
553
579
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
580
|
+
}
|
|
581
|
+
setRam := func(v, ctx) {
|
|
582
|
+
if execFormula.isFormula(v) {
|
|
583
|
+
memFormulaAst = execFormula.astOf(v)
|
|
584
|
+
memFormulaFallback = undefined
|
|
585
|
+
fb := execFormula.fallbackOf(v)
|
|
586
|
+
if !is_undefined(fb) { _assertValidMemRequest(fb, ctx + " ram fallback"); memFormulaFallback = fb }
|
|
587
|
+
ramRequest = undefined
|
|
588
|
+
} else {
|
|
589
|
+
_assertValidMemRequest(v, ctx + " ram")
|
|
590
|
+
ramRequest = v
|
|
591
|
+
memFormulaAst = undefined
|
|
592
|
+
memFormulaFallback = undefined
|
|
557
593
|
}
|
|
558
594
|
}
|
|
595
|
+
setVram := func(v, ctx) {
|
|
596
|
+
if execFormula.isFormula(v) {
|
|
597
|
+
gpuFormulaAst = execFormula.astOf(v)
|
|
598
|
+
gpuFormulaFallback = undefined
|
|
599
|
+
fb := execFormula.fallbackOf(v)
|
|
600
|
+
if !is_undefined(fb) { _assertValidGpuRequest(fb, ctx + " vram fallback"); gpuFormulaFallback = fb }
|
|
601
|
+
gpuMemory = undefined
|
|
602
|
+
} else {
|
|
603
|
+
_assertValidGpuRequest(v, ctx + " vram")
|
|
604
|
+
gpuMemory = v
|
|
605
|
+
gpuFormulaAst = undefined
|
|
606
|
+
gpuFormulaFallback = undefined
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
assertKeys := func(block, name, allowVram) {
|
|
611
|
+
ll.assert(ll.isMap(block), "exec.builder.resources.%s: expected a map", name)
|
|
612
|
+
for k, _ in block {
|
|
613
|
+
ok := k == "cpu" || k == "ram" || (allowVram && k == "vram")
|
|
614
|
+
ll.assert(ok, "exec.builder.resources.%s: unknown key %q (allowed: %s)", name, k, allowVram ? "cpu, ram, vram" : "cpu, ram")
|
|
615
|
+
}
|
|
616
|
+
ll.assert(!is_undefined(block.cpu) && !is_undefined(block.ram),
|
|
617
|
+
"exec.builder.resources.%s: cpu and ram are required", name)
|
|
618
|
+
if allowVram {
|
|
619
|
+
ll.assert(!is_undefined(block.vram),
|
|
620
|
+
"exec.builder.resources.%s: vram is required (it is what makes this a GPU allocation)", name)
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
if hasCPU { assertKeys(spec.onCPU, "onCPU", false) }
|
|
625
|
+
if hasGPU { assertKeys(spec.onGPU, "onGPU", true) }
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
useGPU := hasGPU && (!hasCPU || feats.hasGpu)
|
|
634
|
+
if hasGPU && !hasCPU {
|
|
635
|
+
ll.assert(feats.hasGpu,
|
|
636
|
+
"exec.builder.resources: onGPU was set without onCPU (GPU required), but this backend has no GPU support (exec.hasGpu is false). Add an onCPU block for an adaptive CPU fallback, or run on a GPU-enabled backend.")
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
if useGPU {
|
|
640
|
+
g := spec.onGPU
|
|
641
|
+
setCpu(g.cpu, "resources.onGPU")
|
|
642
|
+
setRam(g.ram, "resources.onGPU")
|
|
643
|
+
setVram(g.vram, "resources.onGPU")
|
|
644
|
+
} else {
|
|
645
|
+
c := spec.onCPU
|
|
646
|
+
setCpu(c.cpu, "resources.onCPU")
|
|
647
|
+
setRam(c.ram, "resources.onCPU")
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
gpuMemory = undefined
|
|
651
|
+
gpuFormulaAst = undefined
|
|
652
|
+
gpuFormulaFallback = undefined
|
|
653
|
+
}
|
|
559
654
|
return self
|
|
560
655
|
},
|
|
561
656
|
|
|
@@ -567,6 +662,9 @@ builder := func() {
|
|
|
567
662
|
|
|
568
663
|
|
|
569
664
|
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
|
|
570
668
|
gpuMemory: func(...args) {
|
|
571
669
|
ll.assert(len(args) <= 1, "exec.builder.gpuMemory: accepts 0 or 1 argument")
|
|
572
670
|
if len(args) == 0 {
|
|
@@ -1235,9 +1333,10 @@ builder := func() {
|
|
|
1235
1333
|
|
|
1236
1334
|
|
|
1237
1335
|
|
|
1238
|
-
hasFormula := !is_undefined(memFormulaAst) || !is_undefined(cpuFormulaAst)
|
|
1336
|
+
hasFormula := !is_undefined(memFormulaAst) || !is_undefined(cpuFormulaAst) || !is_undefined(gpuFormulaAst)
|
|
1239
1337
|
resolvedMem := undefined
|
|
1240
1338
|
resolvedCpu := undefined
|
|
1339
|
+
resolvedGpu := undefined
|
|
1241
1340
|
lineCountMapResource := undefined
|
|
1242
1341
|
|
|
1243
1342
|
if hasFormula {
|
|
@@ -1258,12 +1357,15 @@ builder := func() {
|
|
|
1258
1357
|
|
|
1259
1358
|
fb := execFormula.applyFormulaFallback({
|
|
1260
1359
|
ram: { ast: memFormulaAst, fallback: memFormulaFallback, static: ramRequest },
|
|
1261
|
-
cpu: { ast: cpuFormulaAst, fallback: cpuFormulaFallback, static: cpuRequest }
|
|
1360
|
+
cpu: { ast: cpuFormulaAst, fallback: cpuFormulaFallback, static: cpuRequest },
|
|
1361
|
+
gpu: { ast: gpuFormulaAst, fallback: gpuFormulaFallback, static: gpuMemory }
|
|
1262
1362
|
})
|
|
1263
1363
|
ll.assert(is_undefined(fb.err), fb.err)
|
|
1264
1364
|
ramRequest = fb.ram
|
|
1265
1365
|
cpuRequest = fb.cpu
|
|
1366
|
+
gpuMemory = fb.gpu
|
|
1266
1367
|
if is_int(ramRequest) { ramRequest = string(ramRequest) + "b" }
|
|
1368
|
+
if is_int(gpuMemory) { gpuMemory = string(gpuMemory) + "b" }
|
|
1267
1369
|
} else {
|
|
1268
1370
|
tagMap := {}
|
|
1269
1371
|
for fileName, tag in fileTags {
|
|
@@ -1283,6 +1385,11 @@ builder := func() {
|
|
|
1283
1385
|
refs := execFormula.referencedMetricFiles(resolvedCpu)
|
|
1284
1386
|
if !is_undefined(refs.lineCount) { sets.add(neededLineCount, refs.lineCount...) }
|
|
1285
1387
|
}
|
|
1388
|
+
if !is_undefined(gpuFormulaAst) {
|
|
1389
|
+
resolvedGpu = execFormula.resolveAst(gpuFormulaAst, tagMap, allFiles)
|
|
1390
|
+
refs := execFormula.referencedMetricFiles(resolvedGpu)
|
|
1391
|
+
if !is_undefined(refs.lineCount) { sets.add(neededLineCount, refs.lineCount...) }
|
|
1392
|
+
}
|
|
1286
1393
|
|
|
1287
1394
|
if len(neededLineCount) > 0 {
|
|
1288
1395
|
perFile := {}
|
|
@@ -1299,16 +1406,18 @@ builder := func() {
|
|
|
1299
1406
|
cpu: cpuRequest,
|
|
1300
1407
|
ram: ramRequest
|
|
1301
1408
|
}
|
|
1302
|
-
if !is_undefined(gpuMemory) {
|
|
1409
|
+
if !is_undefined(gpuMemory) || !is_undefined(gpuFormulaAst) {
|
|
1303
1410
|
|
|
1304
1411
|
|
|
1305
1412
|
|
|
1306
1413
|
|
|
1307
1414
|
|
|
1308
1415
|
|
|
1309
|
-
ll.assert(!is_undefined(cpuRequest) || !is_undefined(cpuFormulaAst), "exec.builder: cpu
|
|
1310
|
-
ll.assert(!is_undefined(ramRequest) || !is_undefined(memFormulaAst), "exec.builder:
|
|
1311
|
-
|
|
1416
|
+
ll.assert(!is_undefined(cpuRequest) || !is_undefined(cpuFormulaAst), "exec.builder: a cpu request is required when a GPU is requested. GPU nodes vary in CPU capacity and cost; please provide an explicit CPU request.")
|
|
1417
|
+
ll.assert(!is_undefined(ramRequest) || !is_undefined(memFormulaAst), "exec.builder: a ram request is required when a GPU is requested. GPU nodes vary in RAM capacity and cost; please provide an explicit RAM request.")
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
if !is_undefined(gpuMemory) { quotaData.gpuMemory = gpuMemory }
|
|
1312
1421
|
}
|
|
1313
1422
|
metaExecInputs := {
|
|
1314
1423
|
quota: smart.createJsonResource(quotaData)
|
|
@@ -1332,6 +1441,7 @@ builder := func() {
|
|
|
1332
1441
|
|
|
1333
1442
|
if !is_undefined(resolvedMem) { metaExecInputs[execConstants.META_INPUT_MEM_FORMULA] = smart.createJsonResource(resolvedMem) }
|
|
1334
1443
|
if !is_undefined(resolvedCpu) { metaExecInputs[execConstants.META_INPUT_CPU_FORMULA] = smart.createJsonResource(resolvedCpu) }
|
|
1444
|
+
if !is_undefined(resolvedGpu) { metaExecInputs[execConstants.META_INPUT_GPU_FORMULA] = smart.createJsonResource(resolvedGpu) }
|
|
1335
1445
|
if !is_undefined(lineCountMapResource) { pureExecInputs[execConstants.REGULAR_INPUT_LINE_COUNT] = lineCountMapResource }
|
|
1336
1446
|
|
|
1337
1447
|
|
|
@@ -1528,6 +1638,7 @@ export ll.toStrict({
|
|
|
1528
1638
|
|
|
1529
1639
|
|
|
1530
1640
|
|
|
1641
|
+
|
|
1531
1642
|
hasGpu: feats.hasGpu,
|
|
1532
1643
|
formula: execFormula
|
|
1533
1644
|
})
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platforma-sdk/workflow-tengo",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.7.0",
|
|
4
4
|
"description": "Tengo SDK library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -19,11 +19,11 @@
|
|
|
19
19
|
"@milaboratories/pframes-rs-wasip2": "1.1.52",
|
|
20
20
|
"@milaboratories/software-pframes-conv": "2.2.9",
|
|
21
21
|
"@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
|
|
22
|
-
"@platforma-open/milaboratories.software-
|
|
23
|
-
"@platforma-open/milaboratories.software-
|
|
22
|
+
"@platforma-open/milaboratories.software-ptabler": "2.1.4",
|
|
23
|
+
"@platforma-open/milaboratories.software-ptexter": "1.2.3"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"@platforma-sdk/tengo-builder": "4.0.
|
|
26
|
+
"@platforma-sdk/tengo-builder": "4.0.15"
|
|
27
27
|
},
|
|
28
28
|
"scripts": {
|
|
29
29
|
"type-check": "pl-tengo check",
|