@things-factory/dataset 5.0.0-alpha.15 → 5.0.0-alpha.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/pages/data-item-list.js +1 -1
- package/client/pages/data-ooc-view.js +134 -0
- package/client/pages/data-ooc.js +54 -4
- package/client/pages/data-sample-view.js +96 -0
- package/client/pages/data-sample.js +19 -1
- package/client/pages/data-set.js +2 -2
- package/dist-server/controllers/create-data-sample.js +104 -0
- package/dist-server/controllers/create-data-sample.js.map +1 -0
- package/dist-server/routes.js +9 -24
- package/dist-server/routes.js.map +1 -1
- package/dist-server/service/data-ooc/data-ooc-subscription.js +65 -0
- package/dist-server/service/data-ooc/data-ooc-subscription.js.map +1 -0
- package/dist-server/service/data-ooc/data-ooc.js +7 -0
- package/dist-server/service/data-ooc/data-ooc.js.map +1 -1
- package/dist-server/service/data-ooc/index.js +2 -1
- package/dist-server/service/data-ooc/index.js.map +1 -1
- package/dist-server/service/data-sample/data-sample-mutation.js +2 -80
- package/dist-server/service/data-sample/data-sample-mutation.js.map +1 -1
- package/dist-server/service/data-sample/data-sample.js +10 -3
- package/dist-server/service/data-sample/data-sample.js.map +1 -1
- package/package.json +14 -14
- package/server/controllers/create-data-sample.ts +143 -0
- package/server/routes.ts +18 -32
- package/server/service/data-ooc/data-ooc-subscription.ts +51 -0
- package/server/service/data-ooc/data-ooc.ts +6 -0
- package/server/service/data-ooc/index.ts +2 -1
- package/server/service/data-sample/data-sample-mutation.ts +2 -105
- package/server/service/data-sample/data-sample.ts +9 -4
- package/translations/en.json +3 -1
- package/translations/ko.json +3 -1
- package/translations/ms.json +4 -2
- package/translations/zh.json +4 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-ooc.js","sourceRoot":"","sources":["../../../server/service/data-ooc/data-ooc.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,qCAWgB;AAChB,iDAA4D;AAC5D,+CAA2E;AAE3E,4DAAuD;AACvD,mDAA8C;AAC9C,yDAAgD;AAChD,6CAA4C;AAE5C,MAAM,SAAS,GAAG,YAAM,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AAC7C,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAA;AAEpC,IAAY,aAOX;AAPD,WAAY,aAAa;IACvB,oCAAmB,CAAA;IACnB,sCAAqB,CAAA;IACrB,wCAAuB,CAAA;IACvB,wCAAuB,CAAA;IACvB,sCAAqB,CAAA;IACrB,sCAAqB,CAAA;AACvB,CAAC,EAPW,aAAa,GAAb,qBAAa,KAAb,qBAAa,QAOxB;AAED,IAAA,+BAAgB,EAAC,aAAa,EAAE;IAC9B,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,+DAA+D;CAC7E,CAAC,CAAA;AAOF,IAAa,OAAO,GAApB,MAAa,OAAO;
|
|
1
|
+
{"version":3,"file":"data-ooc.js","sourceRoot":"","sources":["../../../server/service/data-ooc/data-ooc.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,qCAWgB;AAChB,iDAA4D;AAC5D,+CAA2E;AAE3E,4DAAuD;AACvD,mDAA8C;AAC9C,yDAAgD;AAChD,6CAA4C;AAE5C,MAAM,SAAS,GAAG,YAAM,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AAC7C,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAA;AAEpC,IAAY,aAOX;AAPD,WAAY,aAAa;IACvB,oCAAmB,CAAA;IACnB,sCAAqB,CAAA;IACrB,wCAAuB,CAAA;IACvB,wCAAuB,CAAA;IACvB,sCAAqB,CAAA;IACrB,sCAAqB,CAAA;AACvB,CAAC,EAPW,aAAa,GAAb,qBAAa,KAAb,qBAAa,QAOxB;AAED,IAAA,+BAAgB,EAAC,aAAa,EAAE;IAC9B,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,+DAA+D;CAC7E,CAAC,CAAA;AAOF,IAAa,OAAO,GAApB,MAAa,OAAO;CAuJnB,CAAA;AApJC;IAFC,IAAA,gCAAsB,EAAC,MAAM,CAAC;IAC9B,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,iBAAE,CAAC;;mCACC;AAInB;IAFC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,cAAM,CAAC;IACzB,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACjB,cAAM,oBAAN,cAAM;uCAAA;AAGf;IADC,IAAA,oBAAU,EAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;;yCAChC;AAIjB;IAFC,IAAA,gBAAM,GAAE;IACR,IAAA,oBAAK,GAAE;;qCACI;AAMZ;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;4CACN;AAIpB;IAFC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAO,CAAC;IAC1B,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACjC,kBAAO;wCAAA;AAGjB;IADC,IAAA,oBAAU,EAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;;0CAChC;AAKlB;IAHC,IAAA,kBAAQ,EAAC,IAAI,CAAC,EAAE,CAAC,wBAAU,CAAC;IAC5B,IAAA,oBAAU,GAAE;IACZ,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,wBAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACjC,wBAAU;2CAAA;AAGvB;IADC,IAAA,oBAAU,EAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;;6CAChC;AAMrB;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;oCACb;AAMb;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;oCACb;AAMb;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;sCACL;AAYrB;IAVC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;QACd,IAAI,EACF,aAAa,IAAI,OAAO,IAAI,aAAa,IAAI,SAAS;YACpD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,IAAI,QAAQ;gBAC3B,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS;KAChB,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;iDACD;AAMzB;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;qCACb;AAMb;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;wCACV;AAIhB;IAFC,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,oBAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDAChC,oBAAY,oBAAZ,oBAAY;8CAAA;AAI5B;IAFC,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,oBAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACzC,oBAAY,oBAAZ,oBAAY;qCAAA;AAInB;IAFC,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,oBAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACzC,oBAAY,oBAAZ,oBAAY;qCAAA;AAYnB;IAVC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;QACd,IAAI,EACF,aAAa,IAAI,OAAO,IAAI,aAAa,IAAI,SAAS;YACpD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,IAAI,QAAQ;gBAC3B,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS;KAChB,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;wCACV;AAMhB;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;uCACX;AAIf;IAFC,IAAA,gBAAM,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC1B,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACZ,IAAI;4CAAA;AAIlB;IAFC,IAAA,gBAAM,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC1B,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACZ,IAAI;4CAAA;AAIlB;IAFC,IAAA,0BAAgB,GAAE;IAClB,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACd,IAAI;0CAAA;AAIhB;IAFC,IAAA,0BAAgB,GAAE;IAClB,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACd,IAAI;0CAAA;AAMhB;IAJC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,gBAAI,EAAE;QACvB,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACd,gBAAI,oBAAJ,gBAAI;0CAAA;AAGhB;IADC,IAAA,oBAAU,EAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;;4CAChC;AAMpB;IAJC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,gBAAI,EAAE;QACvB,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDAChB,gBAAI,oBAAJ,gBAAI;wCAAA;AAGd;IADC,IAAA,oBAAU,EAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;;0CAChC;AAMlB;IAJC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,gBAAI,EAAE;QACvB,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDAChB,gBAAI,oBAAJ,gBAAI;wCAAA;AAGd;IADC,IAAA,oBAAU,EAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;;0CAChC;AAtJP,OAAO;IALnB,IAAA,gBAAM,GAAE;IACR,IAAA,eAAK,EAAC,eAAe,EAAE,CAAC,OAAgB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAClG,IAAA,eAAK,EAAC,eAAe,EAAE,CAAC,OAAgB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACpG,IAAA,eAAK,EAAC,eAAe,EAAE,CAAC,OAAgB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACtG,IAAA,yBAAU,EAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,CAAC;GACjD,OAAO,CAuJnB;AAvJY,0BAAO"}
|
|
@@ -4,6 +4,7 @@ exports.resolvers = exports.entities = void 0;
|
|
|
4
4
|
const data_ooc_1 = require("./data-ooc");
|
|
5
5
|
const data_ooc_mutation_1 = require("./data-ooc-mutation");
|
|
6
6
|
const data_ooc_query_1 = require("./data-ooc-query");
|
|
7
|
+
const data_ooc_subscription_1 = require("./data-ooc-subscription");
|
|
7
8
|
exports.entities = [data_ooc_1.DataOoc];
|
|
8
|
-
exports.resolvers = [data_ooc_query_1.DataOocQuery, data_ooc_mutation_1.DataOocMutation];
|
|
9
|
+
exports.resolvers = [data_ooc_query_1.DataOocQuery, data_ooc_mutation_1.DataOocMutation, data_ooc_subscription_1.DataOocSubscription];
|
|
9
10
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../server/service/data-ooc/index.ts"],"names":[],"mappings":";;;AAAA,yCAAoC;AACpC,2DAAqD;AACrD,qDAA+C;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../server/service/data-ooc/index.ts"],"names":[],"mappings":";;;AAAA,yCAAoC;AACpC,2DAAqD;AACrD,qDAA+C;AAC/C,mEAA6D;AAEhD,QAAA,QAAQ,GAAG,CAAC,kBAAO,CAAC,CAAA;AACpB,QAAA,SAAS,GAAG,CAAC,6BAAY,EAAE,mCAAe,EAAE,2CAAmB,CAAC,CAAA"}
|
|
@@ -11,94 +11,16 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
12
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
13
|
};
|
|
14
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
15
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
16
|
-
};
|
|
17
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
15
|
exports.DataSampleMutation = void 0;
|
|
19
16
|
const type_graphql_1 = require("type-graphql");
|
|
20
|
-
const data_item_1 = require("../data-item/data-item");
|
|
21
|
-
const data_set_1 = require("../data-set/data-set");
|
|
22
17
|
const data_sample_1 = require("./data-sample");
|
|
23
18
|
const data_sample_type_1 = require("./data-sample-type");
|
|
24
|
-
const
|
|
25
|
-
const controllers_1 = require("../../controllers");
|
|
26
|
-
const data_ooc_1 = require("../data-ooc/data-ooc");
|
|
19
|
+
const create_data_sample_1 = require("../../controllers/create-data-sample");
|
|
27
20
|
const debug = require('debug')('things-factory:dataset:data-sample:data-sample-mutation');
|
|
28
|
-
// parse variable javascript string pattern
|
|
29
|
-
const replaceVariables = (keys, dic) => {
|
|
30
|
-
for (const k in keys) {
|
|
31
|
-
const matches = keys[k].match(/\$\{\w*\}/g);
|
|
32
|
-
matches &&
|
|
33
|
-
matches.forEach(m => {
|
|
34
|
-
keys[k] = keys[k].replace(m, dic[m.slice(2, -1)]);
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
return keys;
|
|
38
|
-
};
|
|
39
|
-
// It is required UTC date for Partitioning File System like AWS S3 from Athena.
|
|
40
|
-
// ex) %YYYY, %MM, %DD
|
|
41
|
-
const formatDate = (keys, _moment) => {
|
|
42
|
-
for (const k in keys) {
|
|
43
|
-
const matches = keys[k].match(/%\w*/g);
|
|
44
|
-
matches &&
|
|
45
|
-
matches.forEach(m => {
|
|
46
|
-
keys[k] = keys[k].replace(m, _moment.format(m.substr(1)));
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
return keys;
|
|
50
|
-
};
|
|
51
21
|
let DataSampleMutation = class DataSampleMutation {
|
|
52
22
|
async createDataSample(dataSample, context) {
|
|
53
|
-
|
|
54
|
-
const dataSet = await tx.getRepository(data_set_1.DataSet).findOne({
|
|
55
|
-
where: { id: dataSample.dataSet.id }
|
|
56
|
-
});
|
|
57
|
-
const dataItems = await tx.getRepository(data_item_1.DataItem).find({
|
|
58
|
-
where: {
|
|
59
|
-
domain,
|
|
60
|
-
dataSet
|
|
61
|
-
},
|
|
62
|
-
order: {
|
|
63
|
-
sequence: 'DESC'
|
|
64
|
-
}
|
|
65
|
-
});
|
|
66
|
-
const spec = dataItems.reduce((spec, dataItem) => {
|
|
67
|
-
spec[dataItem.tag] = Object.assign(Object.assign({}, dataItem.spec), { name: dataItem.name /* do we need ? */ });
|
|
68
|
-
return spec;
|
|
69
|
-
}, {});
|
|
70
|
-
// appending spec to data for reporting and optimizing AWS Athena search
|
|
71
|
-
dataSample.data = Object.assign(Object.assign({}, dataSample.data), { _spec: JSON.stringify(spec) });
|
|
72
|
-
var partitionKeys = Object.assign({}, dataSet.partitionKeys);
|
|
73
|
-
const collectedAt = dataSample.collectedAt || new Date();
|
|
74
|
-
partitionKeys = formatDate(partitionKeys, (0, moment_1.default)(collectedAt).utc());
|
|
75
|
-
partitionKeys = replaceVariables(partitionKeys, Object.assign({ domain: domain.subdomain, dataSetId: dataSample.dataSet.id }, dataSample.data));
|
|
76
|
-
const { ooc, oos } = controllers_1.DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {};
|
|
77
|
-
const result = await tx.getRepository(data_sample_1.DataSample).save(Object.assign(Object.assign({ name: dataSet.name, description: dataSet.description }, dataSample), { domain,
|
|
78
|
-
partitionKeys,
|
|
79
|
-
spec,
|
|
80
|
-
ooc,
|
|
81
|
-
oos,
|
|
82
|
-
collectedAt, creator: user, updater: user }));
|
|
83
|
-
if (ooc || oos) {
|
|
84
|
-
await tx.getRepository(data_ooc_1.DataOoc).save({
|
|
85
|
-
name: dataSet.name,
|
|
86
|
-
description: dataSet.description,
|
|
87
|
-
dataSet,
|
|
88
|
-
dataSample: result,
|
|
89
|
-
data: dataSample.data,
|
|
90
|
-
domain,
|
|
91
|
-
partitionKeys,
|
|
92
|
-
spec,
|
|
93
|
-
ooc,
|
|
94
|
-
oos,
|
|
95
|
-
state: data_ooc_1.DataOocStatus.CREATED,
|
|
96
|
-
collectedAt,
|
|
97
|
-
creator: user,
|
|
98
|
-
updater: user
|
|
99
|
-
});
|
|
100
|
-
}
|
|
101
|
-
return result;
|
|
23
|
+
return await (0, create_data_sample_1.createDataSample)(dataSample, context);
|
|
102
24
|
}
|
|
103
25
|
};
|
|
104
26
|
__decorate([
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-sample-mutation.js","sourceRoot":"","sources":["../../../server/service/data-sample/data-sample-mutation.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"data-sample-mutation.js","sourceRoot":"","sources":["../../../server/service/data-sample/data-sample-mutation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,+CAAsE;AAEtE,+CAA0C;AAC1C,yDAAkD;AAElD,6EAAuE;AAEvE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,yDAAyD,CAAC,CAAA;AAGzF,IAAa,kBAAkB,GAA/B,MAAa,kBAAkB;IAI7B,KAAK,CAAC,gBAAgB,CAAoB,UAAyB,EAAS,OAAY;QACtF,OAAO,MAAM,IAAA,qCAAgB,EAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IACpD,CAAC;CACF,CAAA;AAHC;IAHC,IAAA,wBAAS,EAAC,sFAAsF,CAAC;IACjG,IAAA,wBAAS,EAAC,cAAc,CAAC;IACzB,IAAA,uBAAQ,EAAC,OAAO,CAAC,EAAE,CAAC,wBAAU,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,CAAC;IACtD,WAAA,IAAA,kBAAG,EAAC,YAAY,CAAC,CAAA;IAA6B,WAAA,IAAA,kBAAG,GAAE,CAAA;;qCAArB,gCAAa;;0DAElE;AANU,kBAAkB;IAD9B,IAAA,uBAAQ,EAAC,wBAAU,CAAC;GACR,kBAAkB,CAO9B;AAPY,gDAAkB"}
|
|
@@ -11,12 +11,12 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
var _a, _b, _c, _d, _e, _f;
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.DataSample = void 0;
|
|
14
|
-
const type_graphql_1 = require("type-graphql");
|
|
15
14
|
const typeorm_1 = require("typeorm");
|
|
16
|
-
const auth_base_1 = require("@things-factory/auth-base");
|
|
17
|
-
const env_1 = require("@things-factory/env");
|
|
18
15
|
const shell_1 = require("@things-factory/shell");
|
|
16
|
+
const type_graphql_1 = require("type-graphql");
|
|
19
17
|
const data_set_1 = require("../data-set/data-set");
|
|
18
|
+
const auth_base_1 = require("@things-factory/auth-base");
|
|
19
|
+
const env_1 = require("@things-factory/env");
|
|
20
20
|
const ORMCONFIG = env_1.config.get('ormconfig', {});
|
|
21
21
|
const DATABASE_TYPE = ORMCONFIG.type;
|
|
22
22
|
let DataSample = class DataSample {
|
|
@@ -54,6 +54,13 @@ __decorate([
|
|
|
54
54
|
(0, type_graphql_1.Field)({ nullable: true }),
|
|
55
55
|
__metadata("design:type", String)
|
|
56
56
|
], DataSample.prototype, "type", void 0);
|
|
57
|
+
__decorate([
|
|
58
|
+
(0, typeorm_1.Column)({
|
|
59
|
+
nullable: true
|
|
60
|
+
}),
|
|
61
|
+
(0, type_graphql_1.Field)({ nullable: true }),
|
|
62
|
+
__metadata("design:type", String)
|
|
63
|
+
], DataSample.prototype, "useCase", void 0);
|
|
57
64
|
__decorate([
|
|
58
65
|
(0, typeorm_1.Column)('simple-json', { nullable: true }),
|
|
59
66
|
(0, type_graphql_1.Field)(type => shell_1.ScalarObject, { nullable: true }),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-sample.js","sourceRoot":"","sources":["../../../server/service/data-sample/data-sample.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,+CAAoD;
|
|
1
|
+
{"version":3,"file":"data-sample.js","sourceRoot":"","sources":["../../../server/service/data-sample/data-sample.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,qCASgB;AAChB,iDAA4D;AAC5D,+CAAoD;AAEpD,mDAA8C;AAC9C,yDAAgD;AAChD,6CAA4C;AAE5C,MAAM,SAAS,GAAG,YAAM,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AAC7C,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAA;AASpC,IAAa,UAAU,GAAvB,MAAa,UAAU;CAgHtB,CAAA;AA7GC;IAFC,IAAA,gCAAsB,EAAC,MAAM,CAAC;IAC9B,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,iBAAE,CAAC;;sCACC;AAInB;IAFC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,cAAM,CAAC;IACzB,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACjB,cAAM,oBAAN,cAAM;0CAAA;AAGf;IADC,IAAA,oBAAU,EAAC,CAAC,UAAsB,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;;4CACzC;AAIjB;IAFC,IAAA,gBAAM,GAAE;IACR,IAAA,oBAAK,GAAE;;wCACI;AAMZ;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;+CACN;AAMpB;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;wCACb;AAMb;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;2CACV;AAIhB;IAFC,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,oBAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDAChC,oBAAY,oBAAZ,oBAAY;iDAAA;AAI5B;IAFC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAO,CAAC;IAC1B,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,kBAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;8BAClC,kBAAO;2CAAA;AAGjB;IADC,IAAA,oBAAU,EAAC,CAAC,UAAsB,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;;6CACzC;AAIlB;IAFC,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,oBAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACzC,oBAAY,oBAAZ,oBAAY;wCAAA;AAInB;IAFC,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzC,IAAA,oBAAK,EAAC,IAAI,CAAC,EAAE,CAAC,oBAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDACzC,oBAAY,oBAAZ,oBAAY;wCAAA;AAYnB;IAVC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;QACd,IAAI,EACF,aAAa,IAAI,OAAO,IAAI,aAAa,IAAI,SAAS;YACpD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,IAAI,QAAQ;gBAC3B,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS;KAChB,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;2CACV;AAMhB;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;0CACX;AAMf;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;uCACb;AAMb;IAJC,IAAA,gBAAM,EAAC;QACN,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;uCACb;AAIb;IAFC,IAAA,gBAAM,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC1B,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACZ,IAAI;+CAAA;AAIlB;IAFC,IAAA,0BAAgB,GAAE;IAClB,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACd,IAAI;6CAAA;AAIhB;IAFC,IAAA,0BAAgB,GAAE;IAClB,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;8BACd,IAAI;6CAAA;AAMhB;IAJC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,gBAAI,EAAE;QACvB,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDAChB,gBAAI,oBAAJ,gBAAI;2CAAA;AAGd;IADC,IAAA,oBAAU,EAAC,CAAC,UAAsB,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;;6CACzC;AAMlB;IAJC,IAAA,mBAAS,EAAC,IAAI,CAAC,EAAE,CAAC,gBAAI,EAAE;QACvB,QAAQ,EAAE,IAAI;KACf,CAAC;IACD,IAAA,oBAAK,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;kDAChB,gBAAI,oBAAJ,gBAAI;2CAAA;AAGd;IADC,IAAA,oBAAU,EAAC,CAAC,UAAsB,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;;6CACzC;AA/GP,UAAU;IARtB,IAAA,gBAAM,GAAE;IACR,IAAA,eAAK,EAAC,kBAAkB,EAAE,CAAC,UAAsB,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACjH,IAAA,eAAK,EACJ,kBAAkB,EAClB,CAAC,UAAsB,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,WAAW,CAAC,EAC3F,EAAE,MAAM,EAAE,KAAK,EAAE,CAClB;IACA,IAAA,yBAAU,EAAC,EAAE,WAAW,EAAE,uBAAuB,EAAE,CAAC;GACxC,UAAU,CAgHtB;AAhHY,gCAAU"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@things-factory/dataset",
|
|
3
|
-
"version": "5.0.0-alpha.
|
|
3
|
+
"version": "5.0.0-alpha.19",
|
|
4
4
|
"main": "dist-server/index.js",
|
|
5
5
|
"browser": "client/index.js",
|
|
6
6
|
"things-factory": true,
|
|
@@ -24,19 +24,19 @@
|
|
|
24
24
|
"migration:create": "node ../../node_modules/typeorm/cli.js migration:create -d ./server/migrations"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
|
-
"@operato/app": "1.0.0-alpha.
|
|
28
|
-
"@operato/data-grist": "1.0.0-alpha.
|
|
29
|
-
"@operato/dataset": "1.0.0-alpha.
|
|
30
|
-
"@operato/graphql": "1.0.0-alpha.
|
|
31
|
-
"@operato/i18n": "1.0.0-alpha.
|
|
32
|
-
"@operato/layout": "1.0.0-alpha.
|
|
33
|
-
"@operato/shell": "1.0.0-alpha.
|
|
34
|
-
"@operato/styles": "1.0.0-alpha.
|
|
35
|
-
"@operato/utils": "1.0.0-alpha.
|
|
36
|
-
"@things-factory/auth-base": "^5.0.0-alpha.
|
|
37
|
-
"@things-factory/env": "^5.0.0-alpha.
|
|
38
|
-
"@things-factory/shell": "^5.0.0-alpha.
|
|
27
|
+
"@operato/app": "1.0.0-alpha.38",
|
|
28
|
+
"@operato/data-grist": "1.0.0-alpha.38",
|
|
29
|
+
"@operato/dataset": "1.0.0-alpha.38",
|
|
30
|
+
"@operato/graphql": "1.0.0-alpha.38",
|
|
31
|
+
"@operato/i18n": "1.0.0-alpha.38",
|
|
32
|
+
"@operato/layout": "1.0.0-alpha.38",
|
|
33
|
+
"@operato/shell": "1.0.0-alpha.38",
|
|
34
|
+
"@operato/styles": "1.0.0-alpha.38",
|
|
35
|
+
"@operato/utils": "1.0.0-alpha.38",
|
|
36
|
+
"@things-factory/auth-base": "^5.0.0-alpha.19",
|
|
37
|
+
"@things-factory/env": "^5.0.0-alpha.19",
|
|
38
|
+
"@things-factory/shell": "^5.0.0-alpha.19",
|
|
39
39
|
"moment": "^2.29.1"
|
|
40
40
|
},
|
|
41
|
-
"gitHead": "
|
|
41
|
+
"gitHead": "2e89b095f6c976bde6f7ecd777dec0f5b900fea3"
|
|
42
42
|
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { DataOoc, DataOocStatus } from '../service/data-ooc/data-ooc'
|
|
2
|
+
import { Domain, getRedirectSubdomainPath, pubsub } from '@things-factory/shell'
|
|
3
|
+
|
|
4
|
+
import { DataItem } from '../service/data-item/data-item'
|
|
5
|
+
import { DataSample } from '../service/data-sample/data-sample'
|
|
6
|
+
import { DataSet } from '../service/data-set/data-set'
|
|
7
|
+
import { DataUseCase } from './data-use-case'
|
|
8
|
+
import { EntityManager } from 'typeorm'
|
|
9
|
+
import { NewDataSample } from '../service/data-sample/data-sample-type'
|
|
10
|
+
import { User } from '@things-factory/auth-base'
|
|
11
|
+
import moment from 'moment'
|
|
12
|
+
|
|
13
|
+
const debug = require('debug')('things-factory:dataset:controller/save-data-sample')
|
|
14
|
+
|
|
15
|
+
// parse variable javascript string pattern
|
|
16
|
+
const replaceVariables = (keys, dic) => {
|
|
17
|
+
for (const k in keys) {
|
|
18
|
+
const matches = keys[k].match(/\$\{\w*\}/g)
|
|
19
|
+
matches &&
|
|
20
|
+
matches.forEach(m => {
|
|
21
|
+
keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
|
|
22
|
+
})
|
|
23
|
+
}
|
|
24
|
+
return keys
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// It is required UTC date for Partitioning File System like AWS S3 from Athena.
|
|
28
|
+
// ex) %YYYY, %MM, %DD
|
|
29
|
+
const formatDate = (keys, _moment) => {
|
|
30
|
+
for (const k in keys) {
|
|
31
|
+
const matches = keys[k].match(/%\w*/g)
|
|
32
|
+
matches &&
|
|
33
|
+
matches.forEach(m => {
|
|
34
|
+
keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
|
|
35
|
+
})
|
|
36
|
+
}
|
|
37
|
+
return keys
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export async function createDataSample(
|
|
41
|
+
dataSample: NewDataSample,
|
|
42
|
+
context: {
|
|
43
|
+
state: {
|
|
44
|
+
domain: Domain
|
|
45
|
+
user: User
|
|
46
|
+
tx: EntityManager
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
): Promise<DataSample> {
|
|
50
|
+
const { domain, user, tx } = context.state
|
|
51
|
+
|
|
52
|
+
const dataSet = await tx.getRepository(DataSet).findOne({
|
|
53
|
+
where: { id: dataSample.dataSet.id }
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
const dataItems = await tx.getRepository(DataItem).find({
|
|
57
|
+
where: {
|
|
58
|
+
domain,
|
|
59
|
+
dataSet
|
|
60
|
+
},
|
|
61
|
+
order: {
|
|
62
|
+
sequence: 'DESC'
|
|
63
|
+
}
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
const spec = dataItems.reduce((spec, dataItem) => {
|
|
67
|
+
spec[dataItem.tag] = {
|
|
68
|
+
...dataItem.spec,
|
|
69
|
+
name: dataItem.name /* do we need ? */
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return spec
|
|
73
|
+
}, {})
|
|
74
|
+
|
|
75
|
+
var partitionKeys = {
|
|
76
|
+
...dataSet.partitionKeys
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const collectedAt = dataSample.collectedAt || new Date()
|
|
80
|
+
partitionKeys = formatDate(partitionKeys, moment(collectedAt).utc())
|
|
81
|
+
partitionKeys = replaceVariables(partitionKeys, {
|
|
82
|
+
domain: domain.subdomain,
|
|
83
|
+
dataSetId: dataSample.dataSet.id,
|
|
84
|
+
...dataSample.data
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
const { ooc, oos } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
|
|
88
|
+
|
|
89
|
+
const result = await tx.getRepository(DataSample).save({
|
|
90
|
+
name: dataSet.name,
|
|
91
|
+
description: dataSet.description,
|
|
92
|
+
useCase: dataSet.useCase,
|
|
93
|
+
...dataSample,
|
|
94
|
+
domain,
|
|
95
|
+
partitionKeys,
|
|
96
|
+
spec,
|
|
97
|
+
ooc,
|
|
98
|
+
oos,
|
|
99
|
+
collectedAt,
|
|
100
|
+
creator: user,
|
|
101
|
+
updater: user
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
if (ooc || oos) {
|
|
105
|
+
const dataOoc = await tx.getRepository(DataOoc).save({
|
|
106
|
+
name: dataSet.name,
|
|
107
|
+
description: dataSet.description,
|
|
108
|
+
useCase: dataSet.useCase,
|
|
109
|
+
dataSet,
|
|
110
|
+
dataSample: result,
|
|
111
|
+
data: dataSample.data,
|
|
112
|
+
domain,
|
|
113
|
+
partitionKeys,
|
|
114
|
+
spec,
|
|
115
|
+
ooc,
|
|
116
|
+
oos,
|
|
117
|
+
state: DataOocStatus.CREATED,
|
|
118
|
+
collectedAt,
|
|
119
|
+
creator: user,
|
|
120
|
+
updater: user
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
debug('dataOoc is about to publish...', dataOoc)
|
|
124
|
+
|
|
125
|
+
pubsub.publish('data-ooc', {
|
|
126
|
+
dataOoc,
|
|
127
|
+
supervisoryRoleId: dataSet.supervisoryRoleId
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
pubsub.publish('notification', {
|
|
131
|
+
notification: {
|
|
132
|
+
domain,
|
|
133
|
+
type: 'error',
|
|
134
|
+
title: `Data OOC occurred on '${dataSet.name}'`,
|
|
135
|
+
body: `Data OOC occurred on '${dataSet.name}'`,
|
|
136
|
+
url: getRedirectSubdomainPath(context, domain.subdomain, `/data-ooc/${dataOoc.id}`),
|
|
137
|
+
timestamp: collectedAt
|
|
138
|
+
}
|
|
139
|
+
})
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
return result
|
|
143
|
+
}
|
package/server/routes.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { getConnection } from 'typeorm'
|
|
1
|
+
import { getConnection, getRepository } from 'typeorm'
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import { User } from '@things-factory/auth-base'
|
|
4
|
+
|
|
5
|
+
import { createDataSample } from './controllers/create-data-sample'
|
|
5
6
|
import { DataSensor } from './service/data-sensor/data-sensor'
|
|
6
7
|
|
|
7
8
|
const debug = require('debug')('things-factory:dataset:routes')
|
|
@@ -47,41 +48,26 @@ process.on('bootstrap-module-global-public-route' as any, (app, globalPublicRout
|
|
|
47
48
|
|
|
48
49
|
const domain = sensor.domain
|
|
49
50
|
const dataSet = sensor.dataSet
|
|
50
|
-
const user =
|
|
51
|
-
|
|
52
|
-
const dataItems = await tx.getRepository(DataItem).find({
|
|
51
|
+
const user: User = await tx.getRepository(User).findOne({
|
|
53
52
|
where: {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
},
|
|
57
|
-
order: {
|
|
58
|
-
sequence: 'DESC'
|
|
53
|
+
reference: sensor.appliance.id,
|
|
54
|
+
userType: 'appliance'
|
|
59
55
|
}
|
|
60
56
|
})
|
|
61
57
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
partitionKeys: dataSet.partitionKeys,
|
|
73
|
-
dataSet,
|
|
74
|
-
data,
|
|
75
|
-
rawData,
|
|
76
|
-
spec,
|
|
77
|
-
source: deviceId,
|
|
78
|
-
collectedAt: new Date(timestamp),
|
|
79
|
-
creator: user,
|
|
80
|
-
updater: user
|
|
81
|
-
})
|
|
58
|
+
return await createDataSample(
|
|
59
|
+
{
|
|
60
|
+
dataSet,
|
|
61
|
+
data,
|
|
62
|
+
rawData,
|
|
63
|
+
source: deviceId,
|
|
64
|
+
collectedAt: new Date(timestamp)
|
|
65
|
+
},
|
|
66
|
+
{ state: { domain, user, tx } }
|
|
67
|
+
)
|
|
82
68
|
})
|
|
83
69
|
|
|
84
|
-
|
|
70
|
+
context.status = 200
|
|
85
71
|
})
|
|
86
72
|
})
|
|
87
73
|
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { Resolver, Subscription, Root } from 'type-graphql'
|
|
2
|
+
import { withFilter } from 'graphql-subscriptions'
|
|
3
|
+
import { pubsub } from '@things-factory/shell'
|
|
4
|
+
import { User } from '@things-factory/auth-base'
|
|
5
|
+
import { DataOoc } from './data-ooc'
|
|
6
|
+
import { getRepository } from 'typeorm'
|
|
7
|
+
|
|
8
|
+
const debug = require('debug')('things-factory:dataset:data-ooc-subscription')
|
|
9
|
+
|
|
10
|
+
@Resolver(DataOoc)
|
|
11
|
+
export class DataOocSubscription {
|
|
12
|
+
@Subscription({
|
|
13
|
+
subscribe: (_, args, context, info) => {
|
|
14
|
+
const { domain, user } = context.state
|
|
15
|
+
const subdomain = domain?.subdomain
|
|
16
|
+
|
|
17
|
+
debug('subscribe', subdomain)
|
|
18
|
+
|
|
19
|
+
if (!domain) {
|
|
20
|
+
throw new Error('domain required')
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (!user.domains?.find(d => d.subdomain === subdomain)) {
|
|
24
|
+
throw new Error(`domain(${subdomain}) is not working for user(${user.email}).`)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
return withFilter(
|
|
28
|
+
() => pubsub.asyncIterator('data-ooc'),
|
|
29
|
+
async (payload, variables, context, info) => {
|
|
30
|
+
const { dataOoc, supervisoryRoleId } = payload
|
|
31
|
+
const { domain } = dataOoc
|
|
32
|
+
|
|
33
|
+
if (subdomain !== domain?.subdomain) {
|
|
34
|
+
return false
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// check if the user has that role
|
|
38
|
+
const userWithRoles: User = await getRepository(User).findOne({
|
|
39
|
+
where: { id: user.id },
|
|
40
|
+
relations: ['roles']
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
return userWithRoles.roles.find(role => role.id === supervisoryRoleId)
|
|
44
|
+
}
|
|
45
|
+
)(_, args, context, info)
|
|
46
|
+
}
|
|
47
|
+
})
|
|
48
|
+
dataOoc(@Root() payload: { dataOoc: DataOoc; supervisoryRoleId: string }): DataOoc {
|
|
49
|
+
return payload.dataOoc
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -113,6 +113,12 @@ export class DataOoc {
|
|
|
113
113
|
@Field({ nullable: true })
|
|
114
114
|
type?: string
|
|
115
115
|
|
|
116
|
+
@Column({
|
|
117
|
+
nullable: true
|
|
118
|
+
})
|
|
119
|
+
@Field({ nullable: true })
|
|
120
|
+
useCase?: string
|
|
121
|
+
|
|
116
122
|
@Column('simple-json', { nullable: true })
|
|
117
123
|
@Field(type => ScalarObject, { nullable: true })
|
|
118
124
|
partitionKeys?: ScalarObject
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DataOoc } from './data-ooc'
|
|
2
2
|
import { DataOocMutation } from './data-ooc-mutation'
|
|
3
3
|
import { DataOocQuery } from './data-ooc-query'
|
|
4
|
+
import { DataOocSubscription } from './data-ooc-subscription'
|
|
4
5
|
|
|
5
6
|
export const entities = [DataOoc]
|
|
6
|
-
export const resolvers = [DataOocQuery, DataOocMutation]
|
|
7
|
+
export const resolvers = [DataOocQuery, DataOocMutation, DataOocSubscription]
|
|
@@ -1,121 +1,18 @@
|
|
|
1
1
|
import { Arg, Ctx, Directive, Mutation, Resolver } from 'type-graphql'
|
|
2
2
|
|
|
3
|
-
import { DataItem } from '../data-item/data-item'
|
|
4
|
-
import { DataSet } from '../data-set/data-set'
|
|
5
3
|
import { DataSample } from './data-sample'
|
|
6
4
|
import { NewDataSample } from './data-sample-type'
|
|
7
5
|
|
|
8
|
-
import
|
|
9
|
-
import { DataUseCase } from '../../controllers'
|
|
10
|
-
import { DataOoc, DataOocStatus } from '../data-ooc/data-ooc'
|
|
6
|
+
import { createDataSample } from '../../controllers/create-data-sample'
|
|
11
7
|
|
|
12
8
|
const debug = require('debug')('things-factory:dataset:data-sample:data-sample-mutation')
|
|
13
9
|
|
|
14
|
-
// parse variable javascript string pattern
|
|
15
|
-
const replaceVariables = (keys, dic) => {
|
|
16
|
-
for (const k in keys) {
|
|
17
|
-
const matches = keys[k].match(/\$\{\w*\}/g)
|
|
18
|
-
matches &&
|
|
19
|
-
matches.forEach(m => {
|
|
20
|
-
keys[k] = keys[k].replace(m, dic[m.slice(2, -1)])
|
|
21
|
-
})
|
|
22
|
-
}
|
|
23
|
-
return keys
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
// It is required UTC date for Partitioning File System like AWS S3 from Athena.
|
|
27
|
-
// ex) %YYYY, %MM, %DD
|
|
28
|
-
const formatDate = (keys, _moment) => {
|
|
29
|
-
for (const k in keys) {
|
|
30
|
-
const matches = keys[k].match(/%\w*/g)
|
|
31
|
-
matches &&
|
|
32
|
-
matches.forEach(m => {
|
|
33
|
-
keys[k] = keys[k].replace(m, _moment.format(m.substr(1)))
|
|
34
|
-
})
|
|
35
|
-
}
|
|
36
|
-
return keys
|
|
37
|
-
}
|
|
38
|
-
|
|
39
10
|
@Resolver(DataSample)
|
|
40
11
|
export class DataSampleMutation {
|
|
41
12
|
@Directive('@privilege(category: "data-sample", privilege: "mutation", domainOwnerGranted: true)')
|
|
42
13
|
@Directive('@transaction')
|
|
43
14
|
@Mutation(returns => DataSample, { description: 'To create new data sample' })
|
|
44
15
|
async createDataSample(@Arg('dataSample') dataSample: NewDataSample, @Ctx() context: any): Promise<DataSample> {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
const dataSet = await tx.getRepository(DataSet).findOne({
|
|
48
|
-
where: { id: dataSample.dataSet.id }
|
|
49
|
-
})
|
|
50
|
-
const dataItems = await tx.getRepository(DataItem).find({
|
|
51
|
-
where: {
|
|
52
|
-
domain,
|
|
53
|
-
dataSet
|
|
54
|
-
},
|
|
55
|
-
order: {
|
|
56
|
-
sequence: 'DESC'
|
|
57
|
-
}
|
|
58
|
-
})
|
|
59
|
-
|
|
60
|
-
const spec = dataItems.reduce((spec, dataItem) => {
|
|
61
|
-
spec[dataItem.tag] = {
|
|
62
|
-
...dataItem.spec,
|
|
63
|
-
name: dataItem.name /* do we need ? */
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
return spec
|
|
67
|
-
}, {})
|
|
68
|
-
|
|
69
|
-
// appending spec to data for reporting and optimizing AWS Athena search
|
|
70
|
-
dataSample.data = { ...dataSample.data, _spec: JSON.stringify(spec) }
|
|
71
|
-
|
|
72
|
-
var partitionKeys = {
|
|
73
|
-
...dataSet.partitionKeys
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
const collectedAt = dataSample.collectedAt || new Date()
|
|
77
|
-
partitionKeys = formatDate(partitionKeys, moment(collectedAt).utc())
|
|
78
|
-
partitionKeys = replaceVariables(partitionKeys, {
|
|
79
|
-
domain: domain.subdomain,
|
|
80
|
-
dataSetId: dataSample.dataSet.id,
|
|
81
|
-
...dataSample.data
|
|
82
|
-
})
|
|
83
|
-
|
|
84
|
-
const { ooc, oos } = DataUseCase.evaluate(dataSet, dataItems, dataSample.data) || {}
|
|
85
|
-
|
|
86
|
-
const result = await tx.getRepository(DataSample).save({
|
|
87
|
-
name: dataSet.name,
|
|
88
|
-
description: dataSet.description,
|
|
89
|
-
...dataSample,
|
|
90
|
-
domain,
|
|
91
|
-
partitionKeys,
|
|
92
|
-
spec,
|
|
93
|
-
ooc,
|
|
94
|
-
oos,
|
|
95
|
-
collectedAt,
|
|
96
|
-
creator: user,
|
|
97
|
-
updater: user
|
|
98
|
-
})
|
|
99
|
-
|
|
100
|
-
if (ooc || oos) {
|
|
101
|
-
await tx.getRepository(DataOoc).save({
|
|
102
|
-
name: dataSet.name,
|
|
103
|
-
description: dataSet.description,
|
|
104
|
-
dataSet,
|
|
105
|
-
dataSample: result,
|
|
106
|
-
data: dataSample.data,
|
|
107
|
-
domain,
|
|
108
|
-
partitionKeys,
|
|
109
|
-
spec,
|
|
110
|
-
ooc,
|
|
111
|
-
oos,
|
|
112
|
-
state: DataOocStatus.CREATED,
|
|
113
|
-
collectedAt,
|
|
114
|
-
creator: user,
|
|
115
|
-
updater: user
|
|
116
|
-
})
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
return result
|
|
16
|
+
return await createDataSample(dataSample, context)
|
|
120
17
|
}
|
|
121
18
|
}
|