@platforma-sdk/workflow-tengo 2.13.1 → 2.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/tengo/lib/exec.constants.lib.tengo +11 -1
- package/dist/tengo/lib/exec.lib.tengo +15 -34
- package/dist/tengo/lib/exec.monetization.lib.tengo +43 -170
- package/dist/tengo/lib/exec.monetization_internal.lib.tengo +266 -0
- package/dist/tengo/lib/slices.lib.tengo +37 -26
- package/dist/tengo/lib/workdir.lib.tengo +6 -0
- package/dist/tengo/tpl/exec.R.create-renv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.map-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
|
|
2
|
-
declare type TplName = "
|
|
2
|
+
declare type TplName = "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.R.create-renv" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "exec.python.create-venv" | "exec.monetization_to_json" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.export-pframe";
|
|
3
3
|
declare const Templates: Record<TplName, TemplateFromFile>;
|
|
4
4
|
export { Templates };
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module.exports = { Templates: {
|
|
2
|
-
'exec.monetization_to_json': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.monetization_to_json.plj.gz') },
|
|
3
2
|
'pframes.export-single-pcolumn': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
|
|
4
3
|
'pframes.import-dir': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.import-dir.plj.gz') },
|
|
5
4
|
'pframes.map-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.map-pframe.plj.gz') },
|
|
@@ -9,6 +8,7 @@ module.exports = { Templates: {
|
|
|
9
8
|
'exec.python.install-deps': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.install-deps.plj.gz') },
|
|
10
9
|
'pframes.export-multiple-pcolumns': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
|
|
11
10
|
'exec.python.create-venv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.create-venv.plj.gz') },
|
|
11
|
+
'exec.monetization_to_json': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.monetization_to_json.plj.gz') },
|
|
12
12
|
'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
|
|
13
13
|
'pframes.xsv-export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
|
|
14
14
|
'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') },
|
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
export const Templates = {
|
|
3
|
-
'exec.monetization_to_json': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.monetization_to_json.plj.gz') },
|
|
4
3
|
'pframes.export-single-pcolumn': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
|
|
5
4
|
'pframes.import-dir': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.import-dir.plj.gz') },
|
|
6
5
|
'pframes.map-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.map-pframe.plj.gz') },
|
|
@@ -10,6 +9,7 @@ export const Templates = {
|
|
|
10
9
|
'exec.python.install-deps': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.install-deps.plj.gz') },
|
|
11
10
|
'pframes.export-multiple-pcolumns': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
|
|
12
11
|
'exec.python.create-venv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.create-venv.plj.gz') },
|
|
12
|
+
'exec.monetization_to_json': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.monetization_to_json.plj.gz') },
|
|
13
13
|
'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
|
|
14
14
|
'pframes.xsv-export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
|
|
15
15
|
'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') },
|
|
@@ -3,9 +3,19 @@ ARG_TYPE_EXPRESSION := "expressionRef"
|
|
|
3
3
|
ARG_TYPE_VAR := "variableRef" // TODO: deprecated, remove after 15 Mar 2025
|
|
4
4
|
ARG_TYPES := [ARG_TYPE_STRING, ARG_TYPE_EXPRESSION, ARG_TYPE_VAR]
|
|
5
5
|
|
|
6
|
+
HEAVY_QUEUE := "heavy"
|
|
7
|
+
MEDIUM_QUEUE := "medium"
|
|
8
|
+
LIGHT_QUEUE := "light"
|
|
9
|
+
UI_TASKS_QUEUE := "ui-tasks"
|
|
10
|
+
|
|
6
11
|
export {
|
|
7
12
|
ARG_TYPE_STRING: ARG_TYPE_STRING,
|
|
8
13
|
ARG_TYPE_EXPRESSION: ARG_TYPE_EXPRESSION,
|
|
9
14
|
ARG_TYPE_VAR: ARG_TYPE_VAR,
|
|
10
|
-
ARG_TYPES: ARG_TYPES
|
|
15
|
+
ARG_TYPES: ARG_TYPES,
|
|
16
|
+
|
|
17
|
+
UI_TASKS_QUEUE: UI_TASKS_QUEUE,
|
|
18
|
+
LIGHT_QUEUE: LIGHT_QUEUE,
|
|
19
|
+
MEDIUM_QUEUE: MEDIUM_QUEUE,
|
|
20
|
+
HEAVY_QUEUE: HEAVY_QUEUE
|
|
11
21
|
}
|
|
@@ -18,14 +18,9 @@ oop := import("@platforma-sdk/workflow-tengo:oop")
|
|
|
18
18
|
validation := import("@platforma-sdk/workflow-tengo:validation")
|
|
19
19
|
constants := import("@platforma-sdk/workflow-tengo:constants")
|
|
20
20
|
limits := import("@platforma-sdk/workflow-tengo:exec.limits")
|
|
21
|
-
monetization := import("@platforma-sdk/workflow-tengo:exec.
|
|
21
|
+
monetization := import("@platforma-sdk/workflow-tengo:exec.monetization_internal")
|
|
22
22
|
execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
|
|
23
23
|
|
|
24
|
-
_HEAVY_QUEUE := "heavy"
|
|
25
|
-
_MEDIUM_QUEUE := "medium"
|
|
26
|
-
_LIGHT_QUEUE := "light"
|
|
27
|
-
_UI_TASKS_QUEUE := "ui-tasks"
|
|
28
|
-
|
|
29
24
|
/**
|
|
30
25
|
* Creates a new builder for constructing and executing commands.
|
|
31
26
|
*
|
|
@@ -54,7 +49,7 @@ builder := func() {
|
|
|
54
49
|
//
|
|
55
50
|
// Limits
|
|
56
51
|
//
|
|
57
|
-
queue :=
|
|
52
|
+
queue := execConstants.HEAVY_QUEUE
|
|
58
53
|
hasLimits := false
|
|
59
54
|
cpuRequest := undefined
|
|
60
55
|
ramRequest := undefined
|
|
@@ -90,25 +85,6 @@ builder := func() {
|
|
|
90
85
|
|
|
91
86
|
mnz := monetization.init()
|
|
92
87
|
|
|
93
|
-
/** Gets mnz key from opts or undefined. */
|
|
94
|
-
_getMnzOption := func(...opts) {
|
|
95
|
-
if slices.length(opts) < 1 {
|
|
96
|
-
return undefined
|
|
97
|
-
}
|
|
98
|
-
opt := opts[0]["mnz"]
|
|
99
|
-
|
|
100
|
-
// readability if: we can return undefined here
|
|
101
|
-
return is_undefined(opt) ? undefined : opt
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/** Adds a monetization option if it's defined into mnzFileParams. */
|
|
105
|
-
_addMnzOption := func(fileName, ...opts) {
|
|
106
|
-
opt := _getMnzOption(opts...)
|
|
107
|
-
if !is_undefined(opt) {
|
|
108
|
-
monetization.addFileParams(mnz, fileName, opt)
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
|
|
112
88
|
stdout := "stdout.txt"
|
|
113
89
|
stderr := "stderr.txt"
|
|
114
90
|
|
|
@@ -242,7 +218,7 @@ builder := func() {
|
|
|
242
218
|
* Execute command in the 'heavy' queue.
|
|
243
219
|
*/
|
|
244
220
|
inHeavyQueue: func() {
|
|
245
|
-
queue =
|
|
221
|
+
queue = execConstants.HEAVY_QUEUE
|
|
246
222
|
return self
|
|
247
223
|
},
|
|
248
224
|
|
|
@@ -250,7 +226,7 @@ builder := func() {
|
|
|
250
226
|
* Execute command in the 'medium' queue.
|
|
251
227
|
*/
|
|
252
228
|
inMediumQueue: func() {
|
|
253
|
-
queue =
|
|
229
|
+
queue = execConstants.MEDIUM_QUEUE
|
|
254
230
|
return self
|
|
255
231
|
},
|
|
256
232
|
|
|
@@ -258,7 +234,7 @@ builder := func() {
|
|
|
258
234
|
* Execute command in the 'light' queue.
|
|
259
235
|
*/
|
|
260
236
|
inLightQueue: func() {
|
|
261
|
-
queue =
|
|
237
|
+
queue = execConstants.LIGHT_QUEUE
|
|
262
238
|
return self
|
|
263
239
|
},
|
|
264
240
|
|
|
@@ -266,7 +242,7 @@ builder := func() {
|
|
|
266
242
|
* Execute command in the 'ui-tasks' queue.
|
|
267
243
|
*/
|
|
268
244
|
inUiQueue: func() {
|
|
269
|
-
queue =
|
|
245
|
+
queue = execConstants.UI_TASKS_QUEUE
|
|
270
246
|
return self
|
|
271
247
|
},
|
|
272
248
|
|
|
@@ -509,8 +485,13 @@ builder := func() {
|
|
|
509
485
|
return self
|
|
510
486
|
},
|
|
511
487
|
|
|
512
|
-
enableMnz: func(
|
|
513
|
-
monetization.enable(mnz,
|
|
488
|
+
enableMnz: func(productKey) {
|
|
489
|
+
monetization.enable(mnz, productKey)
|
|
490
|
+
return self
|
|
491
|
+
},
|
|
492
|
+
|
|
493
|
+
setMnzDryRun: func() {
|
|
494
|
+
monetization.setDryRun(mnz, true)
|
|
514
495
|
return self
|
|
515
496
|
},
|
|
516
497
|
|
|
@@ -526,7 +507,7 @@ builder := func() {
|
|
|
526
507
|
|
|
527
508
|
fileName = path.canonize(fileName)
|
|
528
509
|
filesToAdd[fileName] = file
|
|
529
|
-
|
|
510
|
+
monetization.addOption(mnz, fileName, opts...)
|
|
530
511
|
return self
|
|
531
512
|
},
|
|
532
513
|
|
|
@@ -559,7 +540,7 @@ builder := func() {
|
|
|
559
540
|
fileName = path.canonize(fileName)
|
|
560
541
|
|
|
561
542
|
filesToWrite[fileName] = data
|
|
562
|
-
|
|
543
|
+
monetization.addOption(mnz, fileName, opts...)
|
|
563
544
|
|
|
564
545
|
return self
|
|
565
546
|
},
|
|
@@ -1,189 +1,62 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/** Contains a builder for monetization when we need to run monetization without exec (e.g. in pre-run). */
|
|
2
2
|
|
|
3
3
|
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
validation := import("@platforma-sdk/workflow-tengo:validation")
|
|
7
|
-
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
8
|
-
internal := import("@platforma-sdk/workflow-tengo:exec.internal")
|
|
9
|
-
fmt := import("fmt")
|
|
10
|
-
text := import("text")
|
|
11
|
-
runcmd := import("@platforma-sdk/workflow-tengo:exec.runcmd")
|
|
4
|
+
monetization := import("@platforma-sdk/workflow-tengo:exec.monetization_internal")
|
|
12
5
|
workdir := import("@platforma-sdk/workflow-tengo:workdir")
|
|
13
|
-
|
|
6
|
+
path := import("@platforma-sdk/workflow-tengo:path")
|
|
14
7
|
constants := import("@platforma-sdk/workflow-tengo:exec.constants")
|
|
8
|
+
smart := import("@platforma-sdk/workflow-tengo:smart")
|
|
9
|
+
validation := import("@platforma-sdk/workflow-tengo:validation")
|
|
15
10
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
return ll.toStrict({
|
|
21
|
-
enabled: false,
|
|
22
|
-
|
|
23
|
-
// product name of monetization, e.g. "mixcr_analyze"
|
|
24
|
-
productName: "",
|
|
25
|
-
|
|
26
|
-
// file name to the param, e.g. {fileName: { arg: "dataset", collect: ["size", "lines", "sha256"] }}
|
|
27
|
-
fileParams: {}
|
|
28
|
-
})
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
fileParamsSchema := {
|
|
32
|
-
"arg": "string",
|
|
33
|
-
"collect": [`string,regex=size|lines|sha256`]
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
schema := {
|
|
37
|
-
"enabled": "bool",
|
|
38
|
-
"productName": "string",
|
|
39
|
-
"fileParams": {"any": fileParamsSchema}
|
|
40
|
-
}
|
|
11
|
+
builder := func(productKey) {
|
|
12
|
+
mnz := monetization.init()
|
|
13
|
+
monetization.enable(mnz, productKey)
|
|
14
|
+
monetization.setDryRun(mnz, true)
|
|
41
15
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
_validateParams := func(params) {
|
|
45
|
-
ll.assert(ll.isMap(params), "monetization.validateParams(): mnz params must be a map")
|
|
46
|
-
ll.assert(is_string(params.arg), "monetization.validateParams(): 'arg' in {\"mnz\": {...}} param must be a string")
|
|
47
|
-
ll.assert(is_array(params.collect) && len(params.collect) > 0, "monetization.validateParams(): 'collect' in {\"mnz\": {...}} param must be non-empty an array")
|
|
48
|
-
}
|
|
16
|
+
filesToAdd := {}
|
|
17
|
+
filesToWrite := {}
|
|
49
18
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
ll.assert(len(mnz.fileParams) > 0,
|
|
56
|
-
"monetization.assertIsReady(): at least 1 file param must be set")
|
|
57
|
-
}
|
|
19
|
+
self := undefined
|
|
20
|
+
self = ll.toStrict({
|
|
21
|
+
addFile: func(fileName, file, params) {
|
|
22
|
+
ll.assert(is_string(fileName), "monetization.builder().addFile: fileName must be a string")
|
|
23
|
+
ll.assert(smart.isReference(file), "monetization.builder().addFile: file must be a reference")
|
|
58
24
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
25
|
+
fileName = path.canonize(fileName)
|
|
26
|
+
filesToAdd[fileName] = file
|
|
27
|
+
monetization.addFileParams(mnz, fileName, params)
|
|
28
|
+
return self
|
|
29
|
+
},
|
|
63
30
|
|
|
64
|
-
|
|
65
|
-
|
|
31
|
+
writeFile: func(fileName, data, params) {
|
|
32
|
+
validation.assertType(data, ["or",
|
|
33
|
+
"string",
|
|
34
|
+
"bytes",
|
|
35
|
+
validation.reference])
|
|
66
36
|
|
|
67
|
-
|
|
68
|
-
mnz.productName = productName
|
|
69
|
-
}
|
|
37
|
+
fileName = path.canonize(fileName)
|
|
70
38
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
if mnz.enabled {
|
|
74
|
-
_validateIsReady(mnz)
|
|
75
|
-
return true
|
|
76
|
-
}
|
|
39
|
+
filesToWrite[fileName] = data
|
|
40
|
+
monetization.addFileParams(mnz, fileName, params)
|
|
77
41
|
|
|
78
|
-
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
toJson := func(mnz) {
|
|
82
|
-
validation.assertType(mnz, schema,
|
|
83
|
-
"monetization.toJson(): monetization object does not fit json schema")
|
|
84
|
-
|
|
85
|
-
return ll.fromStrict(mnz)
|
|
86
|
-
}
|
|
42
|
+
return self
|
|
43
|
+
},
|
|
87
44
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
45
|
+
build: func() {
|
|
46
|
+
wd := workdir.builder().
|
|
47
|
+
addFiles(filesToAdd).
|
|
48
|
+
writeFiles(filesToWrite).
|
|
49
|
+
build()
|
|
92
50
|
|
|
93
|
-
|
|
51
|
+
result := monetization.run(mnz, wd, constants.UI_TASKS_QUEUE)
|
|
94
52
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
ll.assert(mnz.enabled,
|
|
99
|
-
"monetization.addFileParams(): monetization must be explicitly enabled by enableMnz(productName) function")
|
|
100
|
-
|
|
101
|
-
ll.assert(is_string(fileName),
|
|
102
|
-
"monetization.addFileParams(): file name must be a string.")
|
|
53
|
+
return result
|
|
54
|
+
}
|
|
55
|
+
})
|
|
103
56
|
|
|
104
|
-
|
|
57
|
+
return self
|
|
105
58
|
}
|
|
106
59
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
* All resources are ephemeral,
|
|
110
|
-
* the deduplication should be above this level.
|
|
111
|
-
*
|
|
112
|
-
* @param: mnz: monetization -- see a beginning of this file.
|
|
113
|
-
* @param: wd: workdir -- workdir where all files from mnz.fileParams already exist. */
|
|
114
|
-
run := func(mnz, wd, queue) {
|
|
115
|
-
_validateIsReady(mnz)
|
|
116
|
-
|
|
117
|
-
mnzSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:mnz-client")
|
|
118
|
-
|
|
119
|
-
builder := runcmd.builder(wd)
|
|
120
|
-
|
|
121
|
-
exprType := feats.commandExpressions ? constants.ARG_TYPE_EXPRESSION : constants.ARG_TYPE_VAR
|
|
122
|
-
|
|
123
|
-
builder.envTyped("MI_LICENSE", exprType, "{secrets.MI_LICENSE}")
|
|
124
|
-
|
|
125
|
-
opts := internal.swToRunCmdOptions(mnzSw.name, mnzSw.version, mnzSw.descriptor)
|
|
126
|
-
// load pkg var that points to mnz-client location.
|
|
127
|
-
for _, varInfo in opts.variables {
|
|
128
|
-
builder.ref(varInfo.refName, varInfo.reference)
|
|
129
|
-
builder.substitutionRule(varInfo.varName, varInfo.refName)
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
builder.cmdTyped(exprType, opts.cmd.value)
|
|
133
|
-
for _, arg in opts.args {
|
|
134
|
-
// now mnz-client doesn't have additional args; this line is just in case.
|
|
135
|
-
builder.argTyped(exprType, arg)
|
|
136
|
-
}
|
|
137
|
-
builder.arg("-productName").arg(mnz.productName)
|
|
138
|
-
|
|
139
|
-
// load file params
|
|
140
|
-
for fileName, param in mnz.fileParams {
|
|
141
|
-
// param is of fileParamsSchema type
|
|
142
|
-
toCollect := text.join(param.collect, ",")
|
|
143
|
-
arg := fmt.sprintf("%s:file:%s:%s", param.arg, fileName, toCollect)
|
|
144
|
-
builder.arg(arg)
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
builder.inQueue(queue)
|
|
148
|
-
stdout := "mnz_stdout.txt"
|
|
149
|
-
builder.stdout(stdout)
|
|
150
|
-
run := builder.build()
|
|
151
|
-
|
|
152
|
-
// save mnz_stdout.txt
|
|
153
|
-
savedWd := workdir.save(run.workdir).
|
|
154
|
-
saveFileContent(stdout).
|
|
155
|
-
build()
|
|
156
|
-
|
|
157
|
-
jwtResultAsBytes := savedWd.filesContent.getFutureInputField(stdout)
|
|
158
|
-
|
|
159
|
-
// runcmd refs gets only json/objects as values,
|
|
160
|
-
// but workdir save returns binary:1 resource type.
|
|
161
|
-
// Hence here we have to convert it to the json's string.
|
|
162
|
-
jwtResultAsString := render.create(
|
|
163
|
-
assets.importTemplate("@platforma-sdk/workflow-tengo:exec.monetization_to_json"),
|
|
164
|
-
{ resource: jwtResultAsBytes }
|
|
165
|
-
).output("jsonedData")
|
|
166
|
-
|
|
167
|
-
return ll.toStrict({
|
|
168
|
-
// other commands should run on this wd,
|
|
169
|
-
// or else they will run concurrently
|
|
170
|
-
// with monetization.
|
|
171
|
-
wdAfter: savedWd.workdir,
|
|
172
|
-
|
|
173
|
-
// mnz-client returns jwt token
|
|
174
|
-
// that we will pass to the next command as env.
|
|
175
|
-
// If we need to change env to a file, that's the place.
|
|
176
|
-
jwtEnvName: _JWT_TOKEN_RESULT_ENV,
|
|
177
|
-
jwtEnvValue: jwtResultAsString
|
|
178
|
-
})
|
|
60
|
+
export {
|
|
61
|
+
builder: builder
|
|
179
62
|
}
|
|
180
|
-
|
|
181
|
-
export ll.toStrict({
|
|
182
|
-
init: init,
|
|
183
|
-
schema: schema,
|
|
184
|
-
enable: enable,
|
|
185
|
-
shouldRun: shouldRun,
|
|
186
|
-
toJson: toJson,
|
|
187
|
-
addFileParams: addFileParams,
|
|
188
|
-
run: run
|
|
189
|
-
})
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
/** The library provides all things to work with a monetization. */
|
|
2
|
+
|
|
3
|
+
base64 := import("base64")
|
|
4
|
+
json := import("json")
|
|
5
|
+
|
|
6
|
+
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
7
|
+
feats := import("@platforma-sdk/workflow-tengo:feats")
|
|
8
|
+
|
|
9
|
+
validation := import("@platforma-sdk/workflow-tengo:validation")
|
|
10
|
+
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
11
|
+
internal := import("@platforma-sdk/workflow-tengo:exec.internal")
|
|
12
|
+
fmt := import("fmt")
|
|
13
|
+
text := import("text")
|
|
14
|
+
runcmd := import("@platforma-sdk/workflow-tengo:exec.runcmd")
|
|
15
|
+
workdir := import("@platforma-sdk/workflow-tengo:workdir")
|
|
16
|
+
render := import("@platforma-sdk/workflow-tengo:render")
|
|
17
|
+
constants := import("@platforma-sdk/workflow-tengo:exec.constants")
|
|
18
|
+
slices := import("@platforma-sdk/workflow-tengo:slices")
|
|
19
|
+
sets := import("@platforma-sdk/workflow-tengo:sets")
|
|
20
|
+
|
|
21
|
+
_JWT_TOKEN_RESULT_ENV := "PLATFORMA_MNZ_JWT"
|
|
22
|
+
|
|
23
|
+
/** mnz-client will print the response from milm to stdout as-is. */
|
|
24
|
+
mnzClientStdoutSchema := ["or",
|
|
25
|
+
// run-spec response
|
|
26
|
+
{ jwtToken: "string" },
|
|
27
|
+
|
|
28
|
+
// dry-run response
|
|
29
|
+
{ mnz: {
|
|
30
|
+
details: {
|
|
31
|
+
spentRuns: "number",
|
|
32
|
+
runsToSpend: "number",
|
|
33
|
+
willRemainAfterRun: "number",
|
|
34
|
+
subscription: {
|
|
35
|
+
availableRuns: "number",
|
|
36
|
+
startsAt: "string",
|
|
37
|
+
expiresAt: "string"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}}
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
/** Just creates a new monetization as a map of params. */
|
|
44
|
+
init := func() {
|
|
45
|
+
return ll.toStrict({
|
|
46
|
+
enabled: false,
|
|
47
|
+
|
|
48
|
+
// product key of monetization
|
|
49
|
+
productKey: "",
|
|
50
|
+
|
|
51
|
+
// just returns a number of runs without the actual calculation.
|
|
52
|
+
// Also, returns a number of remaining runs, a number of total runs etc.
|
|
53
|
+
isDryRun: false,
|
|
54
|
+
|
|
55
|
+
// file name to the param, e.g. {fileName: { arg: "dataset", collect: ["size", "lines", "sha256"] }}
|
|
56
|
+
fileParams: {}
|
|
57
|
+
})
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
fileParamsSchema := {
|
|
61
|
+
"arg": "string",
|
|
62
|
+
"collect": [`string,regex=size|lines|sha256`]
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
schema := {
|
|
66
|
+
"enabled": "bool",
|
|
67
|
+
"productKey": "string",
|
|
68
|
+
"isDryRun": "bool",
|
|
69
|
+
"fileParams": {"any": fileParamsSchema}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Same as validation.assertType(params, fileParamsSchema, ...),
|
|
73
|
+
* but with prettier errors for clients. */
|
|
74
|
+
_validateParams := func(params) {
|
|
75
|
+
ll.assert(ll.isMap(params), "monetization.validateParams(): mnz params must be a map")
|
|
76
|
+
|
|
77
|
+
ll.assert(is_string(params.arg), "monetization.validateParams(): 'arg' in addFileParams must be string")
|
|
78
|
+
|
|
79
|
+
ll.assert(is_array(params.collect) && len(params.collect) > 0, "monetization.validateParams(): 'collect' in {\"mnz\": {...}} param must be non-empty an array")
|
|
80
|
+
|
|
81
|
+
ll.assert(slices.every(params.collect, func(elem) {
|
|
82
|
+
return {
|
|
83
|
+
size: true,
|
|
84
|
+
lines: true,
|
|
85
|
+
sha256: true
|
|
86
|
+
}[elem]
|
|
87
|
+
}), "monetization.validateParams(): params in 'collect' must be either 'size', 'lines' or 'sha256'")
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
_validateIsReady := func(mnz) {
|
|
91
|
+
validation.assertType(mnz, schema,
|
|
92
|
+
"monetization.assertIsReady(): monetization does not fit schema")
|
|
93
|
+
ll.assert(len(mnz.productKey) > 0,
|
|
94
|
+
"monetization.assertIsReady(): product key must be set")
|
|
95
|
+
|
|
96
|
+
// We don't need it, since we could run software without file params
|
|
97
|
+
// ll.assert(len(mnz.fileParams) > 0,
|
|
98
|
+
// "monetization.assertIsReady(): at least 1 file param must be set")
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Enables the monetization for a product. */
|
|
102
|
+
enable := func(mnz, productKey) {
|
|
103
|
+
ll.assert(is_string(productKey),
|
|
104
|
+
"monetization.enable(): product key must be a string, got %v", productKey)
|
|
105
|
+
|
|
106
|
+
validation.assertType(mnz, schema,
|
|
107
|
+
"monetization.enable(): mnz object does not fit schema")
|
|
108
|
+
|
|
109
|
+
mnz.enabled = true
|
|
110
|
+
mnz.productKey = productKey
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
setDryRun := func(mnz, isDryRun) {
|
|
114
|
+
ll.assert(mnz.enabled,
|
|
115
|
+
"monetization.setDryRun(): monetization must be explicitly enabled by enableMnz(productKey) function")
|
|
116
|
+
|
|
117
|
+
ll.assert(is_bool(isDryRun),
|
|
118
|
+
"monetization.setDryRun(): isDryRun must be boolean, got %v", isDryRun)
|
|
119
|
+
|
|
120
|
+
validation.assertType(mnz, schema,
|
|
121
|
+
"monetization.setDryRun(): mnz object does not fit schema")
|
|
122
|
+
|
|
123
|
+
mnz.isDryRun = isDryRun
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Is this monetization enabled and should run? */
|
|
127
|
+
shouldRun := func(mnz) {
|
|
128
|
+
if mnz.enabled {
|
|
129
|
+
_validateIsReady(mnz)
|
|
130
|
+
return true
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return false
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
toJson := func(mnz) {
|
|
137
|
+
validation.assertType(mnz, schema,
|
|
138
|
+
"monetization.toJson(): monetization object does not fit json schema")
|
|
139
|
+
|
|
140
|
+
return ll.fromStrict(mnz)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Adds monetization parameters by a file. */
|
|
144
|
+
addFileParams := func(mnz, fileName, params) {
|
|
145
|
+
validation.assertType(mnz, schema,
|
|
146
|
+
"monetization.addFileParams(): monetization object does not fit json schema")
|
|
147
|
+
|
|
148
|
+
_validateParams(params)
|
|
149
|
+
|
|
150
|
+
validation.assertType(params, fileParamsSchema,
|
|
151
|
+
"monetization.addFileParams(): file params does not fit json schema")
|
|
152
|
+
|
|
153
|
+
ll.assert(mnz.enabled,
|
|
154
|
+
"monetization.addFileParams(): monetization must be explicitly enabled by enableMnz(productKey) function")
|
|
155
|
+
|
|
156
|
+
ll.assert(is_string(fileName),
|
|
157
|
+
"monetization.addFileParams(): file name must be a string.")
|
|
158
|
+
|
|
159
|
+
mnz.fileParams[fileName] = params
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Runs monetization binary on a workdir with all files, and returns a new workdir and a resulted jwt token.
|
|
164
|
+
* All resources are ephemeral,
|
|
165
|
+
* the deduplication should be above this level.
|
|
166
|
+
*
|
|
167
|
+
* @param: mnz: monetization -- see a beginning of this file.
|
|
168
|
+
* @param: wd: workdir -- workdir where all files from mnz.fileParams already exist.
|
|
169
|
+
* @param: queue: queue - in which queue we should run monetization. */
|
|
170
|
+
run := func(mnz, wd, queue) {
|
|
171
|
+
_validateIsReady(mnz)
|
|
172
|
+
|
|
173
|
+
exprType := feats.commandExpressions ? constants.ARG_TYPE_EXPRESSION : constants.ARG_TYPE_VAR
|
|
174
|
+
|
|
175
|
+
mnzSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:mnz-client")
|
|
176
|
+
builder := runcmd.builder(wd)
|
|
177
|
+
builder.envTyped("MI_LICENSE", exprType, "{secrets.MI_LICENSE}")
|
|
178
|
+
|
|
179
|
+
opts := internal.swToRunCmdOptions(mnzSw.name, mnzSw.version, mnzSw.descriptor)
|
|
180
|
+
// load pkg var that points to mnz-client location.
|
|
181
|
+
for _, varInfo in opts.variables {
|
|
182
|
+
builder.ref(varInfo.refName, varInfo.reference)
|
|
183
|
+
builder.substitutionRule(varInfo.varName, varInfo.refName)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
builder.cmdTyped(exprType, opts.cmd.value)
|
|
187
|
+
for _, arg in opts.args {
|
|
188
|
+
// now mnz-client doesn't have additional args; this line is just in case.
|
|
189
|
+
builder.argTyped(exprType, arg)
|
|
190
|
+
}
|
|
191
|
+
if mnz.isDryRun {
|
|
192
|
+
builder.arg("-dry-run")
|
|
193
|
+
}
|
|
194
|
+
builder.arg("-productKey").arg(mnz.productKey)
|
|
195
|
+
|
|
196
|
+
// load file params
|
|
197
|
+
for fileName, param in mnz.fileParams {
|
|
198
|
+
// param is of fileParamsSchema type
|
|
199
|
+
toCollect := text.join(param.collect, ",")
|
|
200
|
+
arg := fmt.sprintf("%s:file:%s:%s", param.arg, fileName, toCollect)
|
|
201
|
+
builder.arg(arg)
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
builder.inQueue(queue)
|
|
205
|
+
stdoutName := "mnz_stdout.txt"
|
|
206
|
+
builder.stdout(stdoutName)
|
|
207
|
+
run := builder.build()
|
|
208
|
+
|
|
209
|
+
// save mnz_stdout.txt
|
|
210
|
+
savedWd := workdir.save(run.workdir).
|
|
211
|
+
saveFileContent(stdoutName).
|
|
212
|
+
build()
|
|
213
|
+
|
|
214
|
+
stdout := savedWd.filesContent.getFutureInputField(stdoutName)
|
|
215
|
+
|
|
216
|
+
result := render.createEphemeral(
|
|
217
|
+
assets.importTemplate("@platforma-sdk/workflow-tengo:exec.monetization_to_json"),
|
|
218
|
+
{ resource: stdout }
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
return ll.toStrict({
|
|
222
|
+
// other commands should run on this wd,
|
|
223
|
+
// or else they will run concurrently
|
|
224
|
+
// with monetization.
|
|
225
|
+
wdAfter: savedWd.workdir,
|
|
226
|
+
|
|
227
|
+
// mnz-client returns jwt token
|
|
228
|
+
// that we will pass to the next command as env.
|
|
229
|
+
jwtEnvName: _JWT_TOKEN_RESULT_ENV,
|
|
230
|
+
jwtEnvValue: result.output("jwtToken"),
|
|
231
|
+
|
|
232
|
+
info: result.output("info")
|
|
233
|
+
})
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Gets mnz key from opts or undefined. */
|
|
237
|
+
_extractOption := func(...opts) {
|
|
238
|
+
if slices.length(opts) < 1 {
|
|
239
|
+
return undefined
|
|
240
|
+
}
|
|
241
|
+
opt := opts[0]["mnz"]
|
|
242
|
+
|
|
243
|
+
// readability if: we can return undefined here
|
|
244
|
+
return is_undefined(opt) ? undefined : opt
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Adds a monetization option if it's defined into mnzFileParams. */
|
|
248
|
+
addOption := func(mnz, fileName, ...opts) {
|
|
249
|
+
opt := _extractOption(opts...)
|
|
250
|
+
if !is_undefined(opt) {
|
|
251
|
+
addFileParams(mnz, fileName, opt)
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
export ll.toStrict({
|
|
256
|
+
init: init,
|
|
257
|
+
schema: schema,
|
|
258
|
+
enable: enable,
|
|
259
|
+
setDryRun: setDryRun,
|
|
260
|
+
shouldRun: shouldRun,
|
|
261
|
+
toJson: toJson,
|
|
262
|
+
addFileParams: addFileParams,
|
|
263
|
+
addOption: addOption,
|
|
264
|
+
run: run,
|
|
265
|
+
mnzClientStdoutSchema: mnzClientStdoutSchema
|
|
266
|
+
})
|
|
@@ -139,11 +139,11 @@ map := func(slice, fn) {
|
|
|
139
139
|
* a function that gets a result got so far and an element and returns a new result.
|
|
140
140
|
* @return final result: any - the final result
|
|
141
141
|
*/
|
|
142
|
-
reduce := func(slice, fn) {
|
|
142
|
+
reduce := func(slice, fn, ...init) {
|
|
143
143
|
result := undefined
|
|
144
144
|
for i, elem in slice {
|
|
145
145
|
if i == 0 {
|
|
146
|
-
result = elem
|
|
146
|
+
result = len(init) > 0 ? init : elem
|
|
147
147
|
} else {
|
|
148
148
|
result = fn(result, elem)
|
|
149
149
|
}
|
|
@@ -152,6 +152,16 @@ reduce := func(slice, fn) {
|
|
|
152
152
|
return result
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
/** Returns true if the function applied to every element returns true. */
|
|
156
|
+
every := func(slice, fn) {
|
|
157
|
+
for _, elem in slice {
|
|
158
|
+
if !fn(elem) {
|
|
159
|
+
return false
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return true
|
|
163
|
+
}
|
|
164
|
+
|
|
155
165
|
//
|
|
156
166
|
// Simple quick-sort implementations:
|
|
157
167
|
// - with user-provided ordering
|
|
@@ -159,45 +169,45 @@ reduce := func(slice, fn) {
|
|
|
159
169
|
//
|
|
160
170
|
|
|
161
171
|
_quickSortFn := func(arr, left, right, less) {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
172
|
+
if is_undefined(arr) || len(arr) <= 1 || left < 0 || left >= len(arr) || right < 0 || right >= len(arr) || left >= right {
|
|
173
|
+
return arr
|
|
174
|
+
}
|
|
175
|
+
idx := left
|
|
176
|
+
for i := left; i < right; i++ {
|
|
177
|
+
if less(arr[i], arr[right]) {
|
|
168
178
|
temp := arr[i]
|
|
169
179
|
arr[i] = arr[idx]
|
|
170
180
|
arr[idx] = temp
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
181
|
+
idx++
|
|
182
|
+
}
|
|
183
|
+
}
|
|
174
184
|
temp := arr[right]
|
|
175
185
|
arr[right] = arr[idx]
|
|
176
186
|
arr[idx] = temp
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
187
|
+
_quickSortFn(arr, left, idx-1, less)
|
|
188
|
+
_quickSortFn(arr, idx+1, right, less)
|
|
189
|
+
return arr
|
|
180
190
|
}
|
|
181
191
|
|
|
182
192
|
_quickSort := func(arr, left, right) {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
193
|
+
if is_undefined(arr) || len(arr) <= 1 || left < 0 || left >= len(arr) || right < 0 || right >= len(arr) || left >= right {
|
|
194
|
+
return arr
|
|
195
|
+
}
|
|
196
|
+
idx := left
|
|
197
|
+
for i := left; i < right; i++ {
|
|
198
|
+
if arr[i] < arr[right] {
|
|
189
199
|
temp := arr[i]
|
|
190
200
|
arr[i] = arr[idx]
|
|
191
201
|
arr[idx] = temp
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
202
|
+
idx++
|
|
203
|
+
}
|
|
204
|
+
}
|
|
195
205
|
temp := arr[right]
|
|
196
206
|
arr[right] = arr[idx]
|
|
197
207
|
arr[idx] = temp
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
208
|
+
_quickSort(arr, left, idx-1)
|
|
209
|
+
_quickSort(arr, idx+1, right)
|
|
210
|
+
return arr
|
|
201
211
|
}
|
|
202
212
|
|
|
203
213
|
/**
|
|
@@ -309,6 +319,7 @@ export ll.toStrict({
|
|
|
309
319
|
filter: filter,
|
|
310
320
|
map: map,
|
|
311
321
|
reduce: reduce,
|
|
322
|
+
every: every,
|
|
312
323
|
quickSortInPlace: quickSortInPlace,
|
|
313
324
|
quickSort: quickSort,
|
|
314
325
|
quickSortFn: quickSortFn,
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@platforma-sdk/workflow-tengo",
|
|
3
|
-
"version": "2.13.
|
|
3
|
+
"version": "2.13.3",
|
|
4
4
|
"description": "Tengo SDK library",
|
|
5
5
|
"types": "./dist/index.d.ts",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
19
|
"@milaboratories/software-pframes-conv": "2.1.12",
|
|
20
|
-
"@platforma-open/milaboratories.software-small-binaries": "^1.15.
|
|
20
|
+
"@platforma-open/milaboratories.software-small-binaries": "^1.15.13"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
23
|
"@platforma-sdk/tengo-builder": "^1.17.7"
|