@platforma-sdk/workflow-tengo 1.7.6 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -1
- package/dist/index.mjs +2 -1
- package/dist/tengo/lib/assets.lib.tengo +1 -1
- package/dist/tengo/lib/file.internal.lib.tengo +39 -39
- package/dist/tengo/lib/ll.lib.tengo +2 -2
- package/dist/tengo/lib/objects.lib.tengo +12 -12
- package/dist/tengo/lib/pframes.lib.tengo +26 -45
- package/dist/tengo/lib/pframes.pcolumn.lib.tengo +71 -0
- package/dist/tengo/lib/pframes.util.lib.tengo +48 -11
- package/dist/tengo/lib/pframes.xsv.lib.tengo +29 -7
- package/dist/tengo/lib/render.lib.tengo +2 -2
- package/dist/tengo/lib/tpl.lib.tengo +1 -1
- package/dist/tengo/lib/workflow.constants.lib.tengo +2 -2
- package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
- package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.aggregate.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.map-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
- package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
- package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
- package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
- package/package.json +1 -1
- package/dist/tengo/tpl/pframes.import-xsv-map.plj.gz +0 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
|
|
2
|
-
declare type TplName = "pframes.aggregate" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "pframes.export-pframe" | "exec.python.create-venv" | "exec.exec" | "pframes.
|
|
2
|
+
declare type TplName = "pframes.aggregate" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "pframes.export-pframe" | "exec.python.create-venv" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-map";
|
|
3
3
|
declare const Templates: Record<TplName, TemplateFromFile>;
|
|
4
4
|
export { Templates };
|
package/dist/index.js
CHANGED
|
@@ -10,5 +10,6 @@ module.exports = { Templates: {
|
|
|
10
10
|
'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') },
|
|
11
11
|
'exec.python.create-venv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.create-venv.plj.gz') },
|
|
12
12
|
'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
|
|
13
|
-
'pframes.
|
|
13
|
+
'pframes.xsv-export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
|
|
14
|
+
'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') }
|
|
14
15
|
}};
|
package/dist/index.mjs
CHANGED
|
@@ -11,5 +11,6 @@ export const Templates = {
|
|
|
11
11
|
'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') },
|
|
12
12
|
'exec.python.create-venv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.create-venv.plj.gz') },
|
|
13
13
|
'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
|
|
14
|
-
'pframes.
|
|
14
|
+
'pframes.xsv-export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
|
|
15
|
+
'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') }
|
|
15
16
|
};
|
|
@@ -66,7 +66,7 @@ isSoftwareInfo := func(toCheck) {
|
|
|
66
66
|
* @param assetName: string - full name of asset entrypoint, including namespace (NPM package name).
|
|
67
67
|
* Local software entrypoints (within the same package) can omit namespace
|
|
68
68
|
* part and start the name with colon:
|
|
69
|
-
* - "@company/package:some-
|
|
69
|
+
* - "@company/package:some-software"
|
|
70
70
|
* - ":my-local-software"
|
|
71
71
|
*/
|
|
72
72
|
importAsset := func(assetName) {
|
|
@@ -31,18 +31,18 @@ RTYPE_BLOB_DOWNLOAD := { Name: "BlobDownload", Version: "2" }
|
|
|
31
31
|
* 'handle' - reference to resource with 'progress' API.
|
|
32
32
|
*/
|
|
33
33
|
createBlobImportExternal := func(settings) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
validation.assertJsonSchema(settings, {
|
|
35
|
+
"dataURL": "string",
|
|
36
|
+
"dataHeaders,omitempty": [["string"]],
|
|
37
|
+
"dataVersion,omitempty": "string"
|
|
38
|
+
})
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
importer := smart.structBuilder(RTYPE_BLOB_IMPORT_EXTERNAL, json.encode(settings)).lockAndBuild()
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
return ll.toStrict({
|
|
43
|
+
file: importer.getField("blob"),
|
|
44
|
+
handle: importer.getField("handle")
|
|
45
|
+
})
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
/**
|
|
@@ -52,19 +52,19 @@ createBlobImportExternal := func(settings) {
|
|
|
52
52
|
* 'handle' - reference to resource with 'progress' API.
|
|
53
53
|
*/
|
|
54
54
|
createBlobImportInternal := func(settings) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
validation.assertJsonSchema(settings, {
|
|
56
|
+
"path": "string",
|
|
57
|
+
"storageId": "string",
|
|
58
|
+
"maxSpeedBytes,omitempty": "number",
|
|
59
|
+
"dataVersion,omitempty": "string"
|
|
60
|
+
})
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
importer := smart.structBuilder(RTYPE_BLOB_IMPORT_INTERNAL, json.encode(settings)).lockAndBuild()
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
64
|
+
return ll.toStrict({
|
|
65
|
+
file: importer.getField("blob"),
|
|
66
|
+
handle: importer.getField("handle")
|
|
67
|
+
})
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
/**
|
|
@@ -74,28 +74,28 @@ createBlobImportInternal := func(settings) {
|
|
|
74
74
|
* 'handle' - reference to resource with 'progress' and 'upload' APIs.
|
|
75
75
|
*/
|
|
76
76
|
createBlobUpload := func(settings) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
77
|
+
validation.assertJsonSchema(settings, {
|
|
78
|
+
"modificationTime": "string",
|
|
79
|
+
"localPath": "string",
|
|
80
|
+
"pathSignature": "string",
|
|
81
|
+
"sizeBytes": "string"
|
|
82
|
+
})
|
|
83
83
|
|
|
84
|
-
|
|
84
|
+
uploader := smart.structBuilder(RTYPE_BLOB_UPLOAD, json.encode(settings)).lockAndBuild()
|
|
85
85
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
86
|
+
return ll.toStrict({
|
|
87
|
+
file: uploader.getField("blob"),
|
|
88
|
+
handle: uploader.getField("handle")
|
|
89
|
+
})
|
|
90
90
|
}
|
|
91
91
|
|
|
92
92
|
export ll.toStrict({
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
RTYPE_BLOB_IMPORT_EXTERNAL : RTYPE_BLOB_IMPORT_EXTERNAL,
|
|
94
|
+
RTYPE_BLOB_IMPORT_INTERNAL : RTYPE_BLOB_IMPORT_INTERNAL,
|
|
95
|
+
RTYPE_BLOB_UPLOAD : RTYPE_BLOB_UPLOAD,
|
|
96
|
+
RTYPE_BLOB_DOWNLOAD : RTYPE_BLOB_DOWNLOAD,
|
|
97
97
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
createBlobImportExternal : createBlobImportExternal,
|
|
99
|
+
createBlobImportInternal : createBlobImportInternal,
|
|
100
|
+
createBlobUpload : createBlobUpload
|
|
101
101
|
})
|
|
@@ -147,7 +147,7 @@ isFieldId := func(val) {
|
|
|
147
147
|
/**
|
|
148
148
|
* Converts reference ID (resource ID or field ID) to stable string representation, suitable for use as map keys
|
|
149
149
|
*/
|
|
150
|
-
|
|
150
|
+
idToString := func(referenceID) {
|
|
151
151
|
if isFieldId(referenceID) {
|
|
152
152
|
return fmt.sprintf("0x%X/%s", referenceID.ResourceID, referenceID.Name)
|
|
153
153
|
}
|
|
@@ -155,7 +155,7 @@ isFieldId := func(val) {
|
|
|
155
155
|
return fmt.sprintf("0x%X", referenceID)
|
|
156
156
|
}
|
|
157
157
|
panic("ll.idToString: <referenceID> cannot be used as resource or field ID")
|
|
158
|
-
|
|
158
|
+
}
|
|
159
159
|
|
|
160
160
|
/**
|
|
161
161
|
* Returns current template renderer
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Utilities for working with objects
|
|
3
|
-
*/
|
|
2
|
+
* Utilities for working with objects
|
|
3
|
+
*/
|
|
4
4
|
|
|
5
5
|
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
* Deletes entries with undefined values from object
|
|
9
|
-
*/
|
|
8
|
+
* Deletes entries with undefined values from object
|
|
9
|
+
*/
|
|
10
10
|
deleteUndefined := func(obj) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
for k, v in obj {
|
|
12
|
+
if is_undefined(v) {
|
|
13
|
+
delete(obj, k)
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
return obj
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export ll.toStrict({
|
|
21
|
-
|
|
22
|
-
})
|
|
21
|
+
deleteUndefined: deleteUndefined
|
|
22
|
+
})
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
|
|
5
5
|
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
6
6
|
oop := import("@platforma-sdk/workflow-tengo:oop")
|
|
7
|
-
json := import("json")
|
|
8
7
|
smart := import("@platforma-sdk/workflow-tengo:smart")
|
|
9
8
|
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
10
9
|
render := import("@platforma-sdk/workflow-tengo:render")
|
|
@@ -15,70 +14,52 @@ constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
|
|
|
15
14
|
* downloaded from the UI side.
|
|
16
15
|
*/
|
|
17
16
|
exportFrame := func(pf) {
|
|
18
|
-
|
|
17
|
+
return render.createEphemeral(assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-pframe"), { pf: pf }).output("result")
|
|
19
18
|
}
|
|
20
19
|
|
|
21
20
|
/**
|
|
22
21
|
* Export single PColumn data
|
|
23
22
|
*/
|
|
24
23
|
exportColumnData := func(data) {
|
|
25
|
-
|
|
24
|
+
return render.createEphemeral(assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.export-single-pcolumn"), { data: data }).output("result")
|
|
26
25
|
}
|
|
27
26
|
|
|
28
|
-
/**
|
|
27
|
+
/**
|
|
29
28
|
* Creates a builder for p-frame resource.
|
|
30
29
|
*/
|
|
31
30
|
pFrameBuilder := func() {
|
|
32
|
-
|
|
31
|
+
r := smart.structBuilder(constants.RTYPE_P_FRAME)
|
|
33
32
|
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
self := undefined
|
|
34
|
+
self = ll.toStrict(oop.inherit(r, {
|
|
36
35
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
36
|
+
add: func(column, spec, data) {
|
|
37
|
+
if (!is_undefined(spec)) {
|
|
38
|
+
r.createInputField(column + ".spec").setRefOrJson(spec)
|
|
39
|
+
}
|
|
40
|
+
if (!is_undefined(data)) {
|
|
41
|
+
r.createInputField(column + ".data").setRefOrJson(data)
|
|
42
|
+
}
|
|
43
|
+
return self
|
|
44
|
+
},
|
|
46
45
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
46
|
+
build: func() {
|
|
47
|
+
return r.lockAndBuild()
|
|
48
|
+
}
|
|
49
|
+
}))
|
|
50
|
+
return self
|
|
52
51
|
}
|
|
53
52
|
|
|
54
|
-
/**
|
|
53
|
+
/**
|
|
55
54
|
* Creates a builder for a resource map p-column's data.
|
|
56
55
|
*/
|
|
57
56
|
pColumnResourceMapDataBuilder := func(keyLength) {
|
|
58
|
-
|
|
59
|
-
constants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP,
|
|
60
|
-
json.encode({ keyLength: 1 })
|
|
61
|
-
)
|
|
62
|
-
|
|
63
|
-
self := undefined
|
|
64
|
-
self = ll.toStrict(oop.inherit(r, {
|
|
65
|
-
|
|
66
|
-
add: func(key, data) {
|
|
67
|
-
ll.assert(is_array(key), "key must be an array, got ", key)
|
|
68
|
-
r.createInputField(json.encode(key)).setRefOrJson(data)
|
|
69
|
-
return self
|
|
70
|
-
},
|
|
71
|
-
|
|
72
|
-
build: func() {
|
|
73
|
-
return r.lockAndBuild()
|
|
74
|
-
}
|
|
75
|
-
}))
|
|
76
|
-
return self
|
|
57
|
+
ll.panic("pframes.pColumnResourceMapDataBuilder is not supported any more; use pcolumn.resourceMapBuilder")
|
|
77
58
|
}
|
|
78
59
|
|
|
79
60
|
export ll.toStrict({
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
61
|
+
exportFrame: exportFrame,
|
|
62
|
+
pFrameBuilder: pFrameBuilder,
|
|
63
|
+
exportColumnData: exportColumnData,
|
|
64
|
+
pColumnResourceMapDataBuilder: pColumnResourceMapDataBuilder
|
|
84
65
|
})
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Working with a single p-column
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
6
|
+
oop := import("@platforma-sdk/workflow-tengo:oop")
|
|
7
|
+
json := import("json")
|
|
8
|
+
smart := import("@platforma-sdk/workflow-tengo:smart")
|
|
9
|
+
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
10
|
+
render := import("@platforma-sdk/workflow-tengo:render")
|
|
11
|
+
constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Returns true if given resource is a p-column resource map data
|
|
15
|
+
*/
|
|
16
|
+
isResourceMap := func(r) {
|
|
17
|
+
return smart.isResource(r) && r.info().Type.Name == constants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP.Name
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Creates a builder for a resource map p-column's data.
|
|
22
|
+
*
|
|
23
|
+
* @param keyLength: int - key length
|
|
24
|
+
*/
|
|
25
|
+
resourceMapBuilder := func(keyLength) {
|
|
26
|
+
r := smart.structBuilder(
|
|
27
|
+
constants.RTYPE_P_COLUMN_DATA_RESOURCE_MAP,
|
|
28
|
+
json.encode({ keyLength: keyLength })
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
self := undefined
|
|
32
|
+
self = ll.toStrict(oop.inherit(r, {
|
|
33
|
+
|
|
34
|
+
add: func(key, data) {
|
|
35
|
+
ll.assert(is_array(key), "key must be an array, got ", key)
|
|
36
|
+
ll.assert(len(key) == keyLength, "array length does not match specified keyLength", key, keyLength)
|
|
37
|
+
r.createInputField(json.encode(key)).setRefOrJson(data)
|
|
38
|
+
return self
|
|
39
|
+
},
|
|
40
|
+
|
|
41
|
+
build: func() {
|
|
42
|
+
return r.lockAndBuild()
|
|
43
|
+
}
|
|
44
|
+
}))
|
|
45
|
+
return self
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Convert map object to a p-column resource map.
|
|
50
|
+
*
|
|
51
|
+
* @param map - tengo map
|
|
52
|
+
* @param keyLength: int - length of the partitioned key
|
|
53
|
+
* @param keyParser: func - function to parse string map keys and return array of p-column keys
|
|
54
|
+
*/
|
|
55
|
+
fromMap := func(map, keyLength, keyParser) {
|
|
56
|
+
b := resourceMapBuilder(keyLength)
|
|
57
|
+
|
|
58
|
+
for k, v in map {
|
|
59
|
+
arr := keyParser(k)
|
|
60
|
+
ll.assert(is_array(arr), "keyParser should return array")
|
|
61
|
+
b.add(arr, v)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return b.build()
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export ll.toStrict({
|
|
68
|
+
fromMap: fromMap,
|
|
69
|
+
isResourceMap: isResourceMap,
|
|
70
|
+
resourceMapBuilder: resourceMapBuilder
|
|
71
|
+
})
|
|
@@ -35,7 +35,7 @@ _SPEC_AXES_SCHEMA := {
|
|
|
35
35
|
// Should the N/A values (or empty rows) be allowed;
|
|
36
36
|
// default: false
|
|
37
37
|
`allowNA,omitempty`: `bool`,
|
|
38
|
-
// Should the parsing error be interpreted as N/A
|
|
38
|
+
// Should the parsing error be interpreted as N/A;
|
|
39
39
|
// default: false
|
|
40
40
|
`parseErrorToNA,omitempty`: `bool`,
|
|
41
41
|
// Specification of the axis
|
|
@@ -69,7 +69,7 @@ _SPEC_COLUMN_SCHEMA := {
|
|
|
69
69
|
// Should the N/A values (or empty rows) be allowed;
|
|
70
70
|
// default: true
|
|
71
71
|
`allowNA,omitempty`: `bool`,
|
|
72
|
-
// Should the parsing error be interpreted as N/A
|
|
72
|
+
// Should the parsing error be interpreted as N/A;
|
|
73
73
|
// default: false
|
|
74
74
|
`parseErrorToNA,omitempty`: `bool`,
|
|
75
75
|
// Specification of the column
|
|
@@ -106,7 +106,7 @@ _SPEC_INDEX_SCHEMA := {
|
|
|
106
106
|
`parentAxes,omitempty`: [`number`]
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
|
|
109
|
+
PFCONV_IMPORT_CFG_SCHEMA := {
|
|
110
110
|
// Columns would be stored using specified format;
|
|
111
111
|
// default: Binary
|
|
112
112
|
`storageFormat,omitempty` : `string,regex=Binary|Json`,
|
|
@@ -121,16 +121,16 @@ PFCONV_CFG_SCHEMA := {
|
|
|
121
121
|
// Axis of type Int with XSV row numbers (would be the last one);
|
|
122
122
|
// default: do not create additional axis with indexes
|
|
123
123
|
`index,omitempty`: _SPEC_INDEX_SCHEMA,
|
|
124
|
-
// String representing NA values of types Int, Long, Float, Double
|
|
124
|
+
// String representing NA values of types Int, Long, Float, Double;
|
|
125
125
|
// default: NaN
|
|
126
126
|
`naStr,omitempty`: `string`,
|
|
127
|
-
// String representing NA values of types String, Bytes
|
|
127
|
+
// String representing NA values of types String, Bytes;
|
|
128
128
|
// default: null
|
|
129
129
|
`nullStr,omitempty`: `string`,
|
|
130
130
|
// Single ASCII character to be used as separator;
|
|
131
131
|
// default: ',' for .csv, '\t' for .tsv
|
|
132
132
|
`separator,omitempty`: `string`,
|
|
133
|
-
// Interpret empty string as NA value
|
|
133
|
+
// Interpret empty string as NA value;
|
|
134
134
|
// default: true
|
|
135
135
|
`emptyStrAsNA,omitempty`: `bool`,
|
|
136
136
|
// Prefix all column names with given string;
|
|
@@ -145,6 +145,42 @@ PFCONV_CFG_SCHEMA := {
|
|
|
145
145
|
`allowArtificialColumns,omitempty`: `bool`
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
_COLUMN_FILTER_SCHEMA := {
|
|
149
|
+
// Match any of the types listed here;
|
|
150
|
+
// default: type is ignored during matching
|
|
151
|
+
`type,omitempty`: `string,regex=Int|Long|Float|Double|String|Bytes`,
|
|
152
|
+
// Match any of the names listed here;
|
|
153
|
+
// default: name is ignored during matching
|
|
154
|
+
`name,omitempty`: `string`,
|
|
155
|
+
// Match requires all the listed annotations to have provided values;
|
|
156
|
+
// default: no annotation equality checks are performed during matching
|
|
157
|
+
`annotationValue,omitempty`: { any: `string` },
|
|
158
|
+
// Match requires all the listed annotations to match provided regex patterns;
|
|
159
|
+
// default: no annotation pattern matching is performed during matching
|
|
160
|
+
`annotationPattern,omitempty`: { any: `string` }
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
PFCONV_EXPORT_CFG_SCHEMA := {
|
|
164
|
+
// String representing NA values of types Int, Long, Float, Double;
|
|
165
|
+
// default: NaN
|
|
166
|
+
`naStr,omitempty`: `string`,
|
|
167
|
+
// String representing NA and missing values of types String, Bytes;
|
|
168
|
+
// default: null
|
|
169
|
+
`nullStr,omitempty`: `string`,
|
|
170
|
+
// Toggle to export or skip axes and columns of type Bytes;
|
|
171
|
+
// default: false
|
|
172
|
+
`exportBytes,omitempty`: `bool`,
|
|
173
|
+
// Single ASCII character to be used as separator;
|
|
174
|
+
// default: ',' for .csv, '\t' for .tsv
|
|
175
|
+
`separator,omitempty`: `string`,
|
|
176
|
+
// Only columns matching the selector are exported;
|
|
177
|
+
// default: all columns are exported
|
|
178
|
+
`columnSelector,omitempty`: [_COLUMN_FILTER_SCHEMA],
|
|
179
|
+
// During export, columns with equal axes specifications would be joined into single table
|
|
180
|
+
// using the join type specified;
|
|
181
|
+
// default: Full
|
|
182
|
+
`joinType,omitempty` : `string,regex=Full|Inner`
|
|
183
|
+
}
|
|
148
184
|
|
|
149
185
|
/**
|
|
150
186
|
* Get column id from the column spec
|
|
@@ -188,7 +224,7 @@ purifySpec := func(spec) {
|
|
|
188
224
|
}
|
|
189
225
|
newSpec.columns = newCols
|
|
190
226
|
|
|
191
|
-
validation.assertJsonSchema(newSpec,
|
|
227
|
+
validation.assertJsonSchema(newSpec, PFCONV_IMPORT_CFG_SCHEMA)
|
|
192
228
|
|
|
193
229
|
return newSpec
|
|
194
230
|
}
|
|
@@ -217,8 +253,8 @@ purifySpec := func(spec) {
|
|
|
217
253
|
* columnsSpec - map of full column specs by ids from the original cfg
|
|
218
254
|
* }
|
|
219
255
|
*/
|
|
220
|
-
|
|
221
|
-
ll.assert(len(ops) <= 1, "too many options for
|
|
256
|
+
decomposePfconvImportCfg := func(cfg, ...ops) {
|
|
257
|
+
ll.assert(len(ops) <= 1, "too many options for decomposePfconvImportCfg: %v", len(ops))
|
|
222
258
|
|
|
223
259
|
axesSpec := []
|
|
224
260
|
if len(ops) == 1 && !is_undefined(ops[0].additionalAxesSpec) {
|
|
@@ -247,9 +283,10 @@ decomposePfconvCfg := func(cfg, ...ops) {
|
|
|
247
283
|
}
|
|
248
284
|
|
|
249
285
|
export ll.toStrict({
|
|
250
|
-
|
|
286
|
+
PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
|
|
287
|
+
PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
|
|
251
288
|
|
|
252
289
|
xsvColumnId: xsvColumnId,
|
|
253
290
|
purifySpec: purifySpec,
|
|
254
|
-
|
|
291
|
+
decomposePfconvImportCfg: decomposePfconvImportCfg
|
|
255
292
|
})
|
|
@@ -6,8 +6,10 @@ validation := import("@platforma-sdk/workflow-tengo:validation")
|
|
|
6
6
|
constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
|
|
7
7
|
objects := import("@platforma-sdk/workflow-tengo:objects")
|
|
8
8
|
render := import("@platforma-sdk/workflow-tengo:render")
|
|
9
|
+
smart := import("@platforma-sdk/workflow-tengo:smart")
|
|
9
10
|
exec := import("@platforma-sdk/workflow-tengo:exec")
|
|
10
11
|
json := import("json")
|
|
12
|
+
text := import("text")
|
|
11
13
|
util := import("@platforma-sdk/workflow-tengo:pframes.util")
|
|
12
14
|
ll := import("@platforma-sdk/workflow-tengo:ll")
|
|
13
15
|
assets := import("@platforma-sdk/workflow-tengo:assets")
|
|
@@ -84,7 +86,7 @@ getColumnSpec := func(axesSpec, col) {
|
|
|
84
86
|
importFile := func(xsvFile, xsvType, spec, ...ops) {
|
|
85
87
|
ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
|
|
86
88
|
|
|
87
|
-
validation.assertJsonSchema(spec, util.
|
|
89
|
+
validation.assertJsonSchema(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
|
|
88
90
|
|
|
89
91
|
xsvFileName := "file." + xsvType
|
|
90
92
|
|
|
@@ -190,7 +192,7 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
|
|
|
190
192
|
* Imports a map with xsv files into a p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
|
|
191
193
|
* provided by the spec used as map keys). Resulting p-columns may be double-partitioned.
|
|
192
194
|
*
|
|
193
|
-
* @param
|
|
195
|
+
* @param xsvMap: reference - a reference to a p-column resource map with xsv files
|
|
194
196
|
* @param xsvType: string - either csv or tsv
|
|
195
197
|
* @param spec: object - xsv conversion specification
|
|
196
198
|
* @param ops: object - additional options
|
|
@@ -217,17 +219,17 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
|
|
|
217
219
|
*/
|
|
218
220
|
importFileMap := func(xsvMap, xsvType, spec, ...ops) {
|
|
219
221
|
|
|
220
|
-
importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-
|
|
222
|
+
importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-map")
|
|
221
223
|
|
|
222
224
|
ops := renderOps({
|
|
223
|
-
|
|
225
|
+
dataOnly: false,
|
|
224
226
|
splitDataAndSpec: false,
|
|
225
227
|
additionalAxesSpec: [],
|
|
226
228
|
transpose: undefined
|
|
227
|
-
|
|
229
|
+
}, ops...)
|
|
228
230
|
|
|
229
231
|
// spec without any metadata (for caching purposes)
|
|
230
|
-
decomposedSpec := util.
|
|
232
|
+
decomposedSpec := util.decomposePfconvImportCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
|
|
231
233
|
pureSpec := decomposedSpec.purifiedCfg
|
|
232
234
|
columnsSpec := decomposedSpec.columnsSpec
|
|
233
235
|
|
|
@@ -262,7 +264,27 @@ importFileMap := func(xsvMap, xsvType, spec, ...ops) {
|
|
|
262
264
|
return result
|
|
263
265
|
}
|
|
264
266
|
|
|
267
|
+
/**
|
|
268
|
+
* Export p-frame into xsv file
|
|
269
|
+
*
|
|
270
|
+
* @param pf: p-frame - a collection of p-columns
|
|
271
|
+
* @param xsvType: string - "csv" or "tsv"
|
|
272
|
+
* @param params: map - export params
|
|
273
|
+
*/
|
|
274
|
+
exportFrame := func(pf, xsvType, params) {
|
|
275
|
+
exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
|
|
276
|
+
|
|
277
|
+
r := render.create(exportPfTpl, {
|
|
278
|
+
pf: pf,
|
|
279
|
+
xsvType: xsvType,
|
|
280
|
+
params: params
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
return r.output("result")
|
|
284
|
+
}
|
|
285
|
+
|
|
265
286
|
export ll.toStrict({
|
|
266
287
|
importFile: importFile,
|
|
267
|
-
importFileMap: importFileMap
|
|
288
|
+
importFileMap: importFileMap,
|
|
289
|
+
exportFrame: exportFrame
|
|
268
290
|
})
|
|
@@ -58,11 +58,11 @@ createUniversal := func(tpl, isEph, inputs) {
|
|
|
58
58
|
* Returns an output field with a given name
|
|
59
59
|
*/
|
|
60
60
|
output: func(name, ...cache) {
|
|
61
|
-
ll.assert(len(cache) == 0 || is_int(cache[0]), "cache should be a number (
|
|
61
|
+
ll.assert(len(cache) == 0 || is_int(cache[0]), "cache should be a number (use 'times' tengo library), got: %v", cache)
|
|
62
62
|
out := renderer.getFutureOutputField(tplutil.TEMPLATE_OUTPUT_PREFIX + name)
|
|
63
63
|
if len(cache) > 0 && cache[0] > 0 {
|
|
64
64
|
cache = cache[0]
|
|
65
|
-
tx.cacheSetToField(out.id, cache
|
|
65
|
+
tx.cacheSetToField(out.id, cache)
|
|
66
66
|
}
|
|
67
67
|
|
|
68
68
|
return out
|
|
@@ -421,7 +421,7 @@ tpl := func() {
|
|
|
421
421
|
outputsWithValues = sets.add(outputsWithValues, _delegatedOutputs...)
|
|
422
422
|
|
|
423
423
|
ll.assert(sets.fromSlice(self.definedOutputs) == outputsWithValues,
|
|
424
|
-
"tpl.body() callback must return the same list of outputs as were given to 'defineOutputs()'. Want %s, got %s",
|
|
424
|
+
"tpl.body() callback must return the same list of outputs as were given to 'defineOutputs()'. Want %s, got %s",
|
|
425
425
|
self.definedOutputs, sets.toSlice(outputsWithValues))
|
|
426
426
|
|
|
427
427
|
// set result to self outputs
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
Binary file
|