@things-factory/dataset 5.0.0-alpha.20 → 5.0.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
At the early stage, partition keys are desinged for dynamic partitioning for Athena. But It will be required so many AWS Glue crawlers also by each 'data-sets'.
|
|
4
4
|
Now partition keys are fixed with some cases. Default is daily dataset, which has S3 Key like 'domain={domain}/datasetid={datasetid}/year=2022/month=3/day=25'. It will be set default with empty value in 'data-sets'.
|
|
5
5
|
|
|
6
|
-
In another case, it is necessary to divide by hours or minutes and store them. This datetime item must be included in the S3 key. It will be required a new Glue Crawler and a Glue Catalog Table. For example if you need to separate directory hourly, you should add "%H" for 'hour' to partition_keys for this. and also should add a new crawler.
|
|
6
|
+
In another case, it is necessary to divide by hours or minutes and store them. This datetime item must be included in the S3 key. It will be required a new Glue Crawler and a Glue Catalog Table. For example if you need to separate directory hourly, you should add "%H" for 'hour' to partition_keys for this. and also should add a new crawler. And it would be needed a column to know it is daily or hourly or minutely for reporting client side. It's not developed yet.
|
|
7
|
+
Look at the below picture for understanding.
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
|
package/client/pages/data-set.js
CHANGED
|
@@ -3,12 +3,13 @@ import './data-item-list'
|
|
|
3
3
|
import './data-set-importer'
|
|
4
4
|
import './data-entry-form'
|
|
5
5
|
|
|
6
|
-
import { CommonButtonStyles, ScrollbarStyles } from '@operato/styles'
|
|
6
|
+
import { CommonButtonStyles, CommonGristStyles, ScrollbarStyles } from '@operato/styles'
|
|
7
7
|
import { PageView, store } from '@operato/shell'
|
|
8
8
|
import { css, html } from 'lit'
|
|
9
9
|
import { i18next, localize } from '@operato/i18n'
|
|
10
10
|
import { notify, openPopup } from '@operato/layout'
|
|
11
11
|
|
|
12
|
+
import JSON5 from 'json5'
|
|
12
13
|
import { OxDataUseCase } from '@operato/dataset'
|
|
13
14
|
import { client } from '@operato/graphql'
|
|
14
15
|
import { connect } from 'pwa-helpers/connect-mixin'
|
|
@@ -20,36 +21,25 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
20
21
|
static get properties() {
|
|
21
22
|
return {
|
|
22
23
|
active: String,
|
|
23
|
-
gristConfig: Object
|
|
24
|
+
gristConfig: Object,
|
|
25
|
+
filters: Object,
|
|
26
|
+
sorters: Object,
|
|
27
|
+
mode: String
|
|
24
28
|
}
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
static get styles() {
|
|
28
32
|
return [
|
|
29
33
|
ScrollbarStyles,
|
|
34
|
+
CommonGristStyles,
|
|
30
35
|
css`
|
|
31
36
|
:host {
|
|
32
37
|
display: flex;
|
|
33
|
-
flex-direction: column;
|
|
34
38
|
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
ox-grist {
|
|
39
|
-
overflow-y: auto;
|
|
40
|
-
flex: 1;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
#filters {
|
|
44
|
-
display: flex;
|
|
45
|
-
flex-direction: row;
|
|
46
|
-
justify-content: space-between;
|
|
47
|
-
|
|
48
|
-
background-color: white;
|
|
49
|
-
}
|
|
39
|
+
width: 100%;
|
|
50
40
|
|
|
51
|
-
|
|
52
|
-
|
|
41
|
+
--grid-record-emphasized-background-color: red;
|
|
42
|
+
--grid-record-emphasized-color: yellow;
|
|
53
43
|
}
|
|
54
44
|
`
|
|
55
45
|
]
|
|
@@ -87,14 +77,43 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
87
77
|
}
|
|
88
78
|
|
|
89
79
|
render() {
|
|
80
|
+
const mode = this.mode || (isMobileDevice() ? 'LIST' : 'GRID')
|
|
81
|
+
|
|
90
82
|
return html`
|
|
91
83
|
<ox-grist
|
|
92
|
-
.mode=${
|
|
84
|
+
.mode=${mode}
|
|
93
85
|
.config=${this.gristConfig}
|
|
86
|
+
.filters=${this.filters}
|
|
87
|
+
.orders=${this.orders}
|
|
94
88
|
.fetchHandler=${this.fetchHandler.bind(this)}
|
|
95
89
|
>
|
|
96
90
|
<div slot="headroom" id="filters">
|
|
97
|
-
<
|
|
91
|
+
<div id="filters">
|
|
92
|
+
<ox-filters-form></ox-filters-form>
|
|
93
|
+
</div>
|
|
94
|
+
|
|
95
|
+
<div id="sorters">
|
|
96
|
+
Sort
|
|
97
|
+
<mwc-icon
|
|
98
|
+
@click=${e => {
|
|
99
|
+
const target = e.currentTarget
|
|
100
|
+
this.renderRoot.querySelector('#sorter-control').open({
|
|
101
|
+
right: 0,
|
|
102
|
+
top: target.offsetTop + target.offsetHeight
|
|
103
|
+
})
|
|
104
|
+
}}
|
|
105
|
+
>expand_more</mwc-icon
|
|
106
|
+
>
|
|
107
|
+
<ox-popup id="sorter-control">
|
|
108
|
+
<ox-sorters-control> </ox-sorters-control>
|
|
109
|
+
</ox-popup>
|
|
110
|
+
</div>
|
|
111
|
+
|
|
112
|
+
<div id="modes">
|
|
113
|
+
<mwc-icon @click=${() => (this.mode = 'GRID')} ?active=${mode == 'GRID'}>grid_on</mwc-icon>
|
|
114
|
+
<mwc-icon @click=${() => (this.mode = 'LIST')} ?active=${mode == 'LIST'}>format_list_bulleted</mwc-icon>
|
|
115
|
+
<mwc-icon @click=${() => (this.mode = 'CARD')} ?active=${mode == 'CARD'}>apps</mwc-icon>
|
|
116
|
+
</div>
|
|
98
117
|
</div>
|
|
99
118
|
</ox-grist>
|
|
100
119
|
`
|
|
@@ -104,32 +123,12 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
104
123
|
return this.renderRoot.querySelector('ox-grist')
|
|
105
124
|
}
|
|
106
125
|
|
|
107
|
-
// update with url params value
|
|
108
|
-
_updateSearchConfig(lifecycle) {
|
|
109
|
-
// this.searchConfig = this.searchConfig.map(conf => {
|
|
110
|
-
// if (conf.name in lifecycle.params) {
|
|
111
|
-
// conf.value = lifecycle.params[conf.name]
|
|
112
|
-
// } else {
|
|
113
|
-
// delete conf.value
|
|
114
|
-
// }
|
|
115
|
-
// return conf
|
|
116
|
-
// })
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
// set default field value to record with searchConfig
|
|
120
|
-
_setDefaultFieldsValue(fields) {
|
|
121
|
-
// this.searchConfig.forEach(conf => {
|
|
122
|
-
// if (!fields[conf.name] && conf.value) {
|
|
123
|
-
// fields[conf.name] = conf.value
|
|
124
|
-
// }
|
|
125
|
-
// })
|
|
126
|
-
}
|
|
127
|
-
|
|
128
126
|
async pageInitialized(lifecycle) {
|
|
129
|
-
this._updateSearchConfig(lifecycle)
|
|
130
|
-
|
|
131
127
|
this.gristConfig = {
|
|
132
|
-
list: {
|
|
128
|
+
list: {
|
|
129
|
+
fields: ['name', 'description'],
|
|
130
|
+
details: ['schedule', 'active']
|
|
131
|
+
},
|
|
133
132
|
columns: [
|
|
134
133
|
{ type: 'gutter', gutterName: 'sequence' },
|
|
135
134
|
{ type: 'gutter', gutterName: 'row-selector', multiple: true },
|
|
@@ -172,7 +171,6 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
172
171
|
{
|
|
173
172
|
type: 'string',
|
|
174
173
|
name: 'name',
|
|
175
|
-
label: true,
|
|
176
174
|
header: i18next.t('field.name'),
|
|
177
175
|
record: {
|
|
178
176
|
editable: true
|
|
@@ -184,7 +182,6 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
184
182
|
{
|
|
185
183
|
type: 'string',
|
|
186
184
|
name: 'description',
|
|
187
|
-
label: true,
|
|
188
185
|
header: i18next.t('field.description'),
|
|
189
186
|
record: {
|
|
190
187
|
editable: true
|
|
@@ -244,7 +241,17 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
244
241
|
}
|
|
245
242
|
},
|
|
246
243
|
sortable: true,
|
|
247
|
-
filter:
|
|
244
|
+
filter: {
|
|
245
|
+
operator: 'eq',
|
|
246
|
+
options: () => {
|
|
247
|
+
return ['', ...OxDataUseCase.getUseCaseNames()].map(name => {
|
|
248
|
+
return {
|
|
249
|
+
display: name,
|
|
250
|
+
value: name
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
}
|
|
254
|
+
},
|
|
248
255
|
width: 80
|
|
249
256
|
},
|
|
250
257
|
{
|
|
@@ -336,10 +343,28 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
336
343
|
|
|
337
344
|
async pageUpdated(changes, lifecycle) {
|
|
338
345
|
if (this.active) {
|
|
339
|
-
// update with url params value
|
|
340
|
-
this._updateSearchConfig(lifecycle)
|
|
341
346
|
await this.updateComplete
|
|
342
347
|
|
|
348
|
+
var filters = lifecycle.params?.['filters']
|
|
349
|
+
if (filters) {
|
|
350
|
+
try {
|
|
351
|
+
filters = JSON5.parse(filters)
|
|
352
|
+
this.filters = filters
|
|
353
|
+
} catch (e) {
|
|
354
|
+
console.error(`filters parameter parsing error: ${e}`)
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
var sorters = lifecycle.params?.['sorters']
|
|
359
|
+
if (sorters) {
|
|
360
|
+
try {
|
|
361
|
+
sorters = JSON5.parse(sorters)
|
|
362
|
+
this.sorters = sorters
|
|
363
|
+
} catch (e) {
|
|
364
|
+
console.error(`sorters parameter parsing error: ${e}`)
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
343
368
|
this.grist.fetch()
|
|
344
369
|
}
|
|
345
370
|
}
|
|
@@ -456,7 +481,6 @@ export class DataSet extends connect(store)(localize(i18next)(PageView)) {
|
|
|
456
481
|
for (let key in dirtyFields) {
|
|
457
482
|
patchField[key] = dirtyFields[key].after
|
|
458
483
|
}
|
|
459
|
-
this._setDefaultFieldsValue(patchField)
|
|
460
484
|
patchField.cuFlag = patch.__dirty__
|
|
461
485
|
|
|
462
486
|
return patchField
|
|
@@ -57,7 +57,7 @@ async function createDataSample(dataSample, context) {
|
|
|
57
57
|
const momentUtc = (0, moment_1.default)(collectedAt).utc();
|
|
58
58
|
const defaultPartitionKeys = {
|
|
59
59
|
domain: domain.subdomain,
|
|
60
|
-
|
|
60
|
+
datasetid: dataSample.dataSet.id,
|
|
61
61
|
year: momentUtc.format('Y'),
|
|
62
62
|
month: momentUtc.format('M'),
|
|
63
63
|
day: momentUtc.format('D')
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../server/routes.ts"],"names":[],"mappings":";;AAAA,
|
|
1
|
+
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../server/routes.ts"],"names":[],"mappings":";;AAAA,qCAAuC;AAEvC,yDAAgD;AAEhD,yEAAmE;AACnE,mEAA8D;AAE9D,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,+BAA+B,CAAC,CAAA;AAE/D,OAAO,CAAC,EAAE,CAAC,sCAA6C,EAAE,CAAC,GAAG,EAAE,kBAAkB,EAAE,EAAE;IACpF;;;;;OAKG;IAEH,kBAAkB,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC9D,SAAS;QACT,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAA;QAChF,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,cAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;SACpG;QAED,uBAAuB;QACvB,MAAM,IAAA,uBAAa,GAAE,CAAC,WAAW,CAAC,KAAK,EAAC,EAAE,EAAC,EAAE;YAC3C,+BAA+B;YAC/B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,wBAAU,CAAC,CAAC,OAAO,CAAC;gBACxD,KAAK,EAAE,EAAE,QAAQ,EAAE;gBACnB,SAAS,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC;aAC9C,CAAC,CAAA;YAEF,IAAI,CAAC,MAAM,EAAE;gBACX,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,aAAa,CAAC,CAAA;aACvE;YAED,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,iBAAiB,CAAC,CAAA;aACjF;YAED,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,iBAAiB,CAAC,CAAA;aACrF;YAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,QAAQ,iBAAiB,CAAC,CAAA;aACnF;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;YAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAA;YAC9B,MAAM,IAAI,GAAS,MAAM,EAAE,CAAC,aAAa,CAAC,gBAAI,CAAC,CAAC,OAAO,CAAC;gBACtD,KAAK,EAAE;oBACL,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE;oBAC9B,QAAQ,EAAE,WAAW;iBACtB;aACF,CAAC,CAAA;YAEF,OAAO,MAAM,IAAA,qCAAgB,EAC3B;gBACE,OAAO;gBACP,IAAI;gBACJ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,WAAW,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC,EACD,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAChC,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,OAAO,CAAC,MAAM,GAAG,GAAG,CAAA;IACtB,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,OAAO,CAAC,EAAE,CAAC,uCAA8C,EAAE,CAAC,GAAG,EAAE,mBAAmB,EAAE,EAAE;IACtF;;OAEG;AACL,CAAC,CAAC,CAAA;AAEF,OAAO,CAAC,EAAE,CAAC,sCAA6C,EAAE,CAAC,GAAG,EAAE,kBAAkB,EAAE,EAAE;IACpF;;OAEG;AACL,CAAC,CAAC,CAAA;AAEF,OAAO,CAAC,EAAE,CAAC,uCAA8C,EAAE,CAAC,GAAG,EAAE,mBAAmB,EAAE,EAAE;IACtF;;OAEG;AACL,CAAC,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@things-factory/dataset",
|
|
3
|
-
"version": "5.0.0-alpha.
|
|
3
|
+
"version": "5.0.0-alpha.21",
|
|
4
4
|
"main": "dist-server/index.js",
|
|
5
5
|
"browser": "client/index.js",
|
|
6
6
|
"things-factory": true,
|
|
@@ -24,19 +24,19 @@
|
|
|
24
24
|
"migration:create": "node ../../node_modules/typeorm/cli.js migration:create -d ./server/migrations"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
|
-
"@operato/app": "1.0.0-alpha.
|
|
28
|
-
"@operato/data-grist": "1.0.0-alpha.
|
|
29
|
-
"@operato/dataset": "1.0.0-alpha.
|
|
30
|
-
"@operato/graphql": "1.0.0-alpha.
|
|
31
|
-
"@operato/i18n": "1.0.0-alpha.
|
|
32
|
-
"@operato/layout": "1.0.0-alpha.
|
|
33
|
-
"@operato/shell": "1.0.0-alpha.
|
|
34
|
-
"@operato/styles": "1.0.0-alpha.
|
|
35
|
-
"@operato/utils": "1.0.0-alpha.
|
|
36
|
-
"@things-factory/auth-base": "^5.0.0-alpha.
|
|
37
|
-
"@things-factory/env": "^5.0.0-alpha.
|
|
38
|
-
"@things-factory/shell": "^5.0.0-alpha.
|
|
27
|
+
"@operato/app": "1.0.0-alpha.43",
|
|
28
|
+
"@operato/data-grist": "1.0.0-alpha.43",
|
|
29
|
+
"@operato/dataset": "1.0.0-alpha.43",
|
|
30
|
+
"@operato/graphql": "1.0.0-alpha.43",
|
|
31
|
+
"@operato/i18n": "1.0.0-alpha.43",
|
|
32
|
+
"@operato/layout": "1.0.0-alpha.43",
|
|
33
|
+
"@operato/shell": "1.0.0-alpha.43",
|
|
34
|
+
"@operato/styles": "1.0.0-alpha.43",
|
|
35
|
+
"@operato/utils": "1.0.0-alpha.43",
|
|
36
|
+
"@things-factory/auth-base": "^5.0.0-alpha.21",
|
|
37
|
+
"@things-factory/env": "^5.0.0-alpha.21",
|
|
38
|
+
"@things-factory/shell": "^5.0.0-alpha.21",
|
|
39
39
|
"moment": "^2.29.1"
|
|
40
40
|
},
|
|
41
|
-
"gitHead": "
|
|
41
|
+
"gitHead": "4bce1f9b142a4a1d1c786a67415401c96c44eb89"
|
|
42
42
|
}
|
|
@@ -76,7 +76,7 @@ export async function createDataSample(
|
|
|
76
76
|
const momentUtc = moment(collectedAt).utc()
|
|
77
77
|
const defaultPartitionKeys = {
|
|
78
78
|
domain: domain.subdomain,
|
|
79
|
-
|
|
79
|
+
datasetid: dataSample.dataSet.id, /* It should not be 'data_set_id' as column name duplicated for Glue */
|
|
80
80
|
year: momentUtc.format('Y'),
|
|
81
81
|
month: momentUtc.format('M'),
|
|
82
82
|
day: momentUtc.format('D')
|
package/server/routes.ts
CHANGED