@golemio/rush-hour-aggregation 1.0.0-dev.520603218
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +11 -0
- package/README.md +47 -0
- package/db/migrations/postgresql/.config.json +3 -0
- package/db/migrations/postgresql/20220405102916-prahaZasekanaAggregation.js +53 -0
- package/db/migrations/postgresql/package.json +3 -0
- package/db/migrations/postgresql/sqls/20220405102916-prahaZasekanaAggregation-down.sql +6 -0
- package/db/migrations/postgresql/sqls/20220405102916-prahaZasekanaAggregation-up.sql +64 -0
- package/db/migrations/postgresql/sqls/package.json +3 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/integration-engine/AggregationWorker.d.ts +16 -0
- package/dist/integration-engine/AggregationWorker.js +83 -0
- package/dist/integration-engine/AggregationWorker.js.map +1 -0
- package/dist/integration-engine/WorkerHelper.d.ts +9 -0
- package/dist/integration-engine/WorkerHelper.js +52 -0
- package/dist/integration-engine/WorkerHelper.js.map +1 -0
- package/dist/integration-engine/business/AggregationTask.d.ts +11 -0
- package/dist/integration-engine/business/AggregationTask.js +18 -0
- package/dist/integration-engine/business/AggregationTask.js.map +1 -0
- package/dist/integration-engine/business/interfaces/IAggregationTask.d.ts +6 -0
- package/dist/integration-engine/business/interfaces/IAggregationTask.js +3 -0
- package/dist/integration-engine/business/interfaces/IAggregationTask.js.map +1 -0
- package/dist/integration-engine/dataAccess/AggregationRepository.d.ts +13 -0
- package/dist/integration-engine/dataAccess/AggregationRepository.js +60 -0
- package/dist/integration-engine/dataAccess/AggregationRepository.js.map +1 -0
- package/dist/integration-engine/dataAccess/FcdEventsRepository.d.ts +6 -0
- package/dist/integration-engine/dataAccess/FcdEventsRepository.js +38 -0
- package/dist/integration-engine/dataAccess/FcdEventsRepository.js.map +1 -0
- package/dist/integration-engine/dataAccess/NdicEventRepository.d.ts +6 -0
- package/dist/integration-engine/dataAccess/NdicEventRepository.js +38 -0
- package/dist/integration-engine/dataAccess/NdicEventRepository.js.map +1 -0
- package/dist/integration-engine/dataAccess/TskEventsRepository.d.ts +6 -0
- package/dist/integration-engine/dataAccess/TskEventsRepository.js +38 -0
- package/dist/integration-engine/dataAccess/TskEventsRepository.js.map +1 -0
- package/dist/integration-engine/dataAccess/WazeJamsRepository.d.ts +6 -0
- package/dist/integration-engine/dataAccess/WazeJamsRepository.js +40 -0
- package/dist/integration-engine/dataAccess/WazeJamsRepository.js.map +1 -0
- package/dist/integration-engine/dataAccess/WazeReconstructionsRepository.d.ts +6 -0
- package/dist/integration-engine/dataAccess/WazeReconstructionsRepository.js +38 -0
- package/dist/integration-engine/dataAccess/WazeReconstructionsRepository.js.map +1 -0
- package/dist/integration-engine/index.d.ts +2 -0
- package/dist/integration-engine/index.js +16 -0
- package/dist/integration-engine/index.js.map +1 -0
- package/dist/integration-engine/queueDefinitions.d.ts +3 -0
- package/dist/integration-engine/queueDefinitions.js +25 -0
- package/dist/integration-engine/queueDefinitions.js.map +1 -0
- package/dist/integration-engine/service/AggregationFactory.d.ts +7 -0
- package/dist/integration-engine/service/AggregationFactory.js +30 -0
- package/dist/integration-engine/service/AggregationFactory.js.map +1 -0
- package/dist/integration-engine/service/aggregators/AbstractAggregator.d.ts +8 -0
- package/dist/integration-engine/service/aggregators/AbstractAggregator.js +27 -0
- package/dist/integration-engine/service/aggregators/AbstractAggregator.js.map +1 -0
- package/dist/integration-engine/service/aggregators/FcdEventsAggregator.d.ts +12 -0
- package/dist/integration-engine/service/aggregators/FcdEventsAggregator.js +50 -0
- package/dist/integration-engine/service/aggregators/FcdEventsAggregator.js.map +1 -0
- package/dist/integration-engine/service/aggregators/NdicEventsAggregator.d.ts +11 -0
- package/dist/integration-engine/service/aggregators/NdicEventsAggregator.js +57 -0
- package/dist/integration-engine/service/aggregators/NdicEventsAggregator.js.map +1 -0
- package/dist/integration-engine/service/aggregators/TskEventsAggregator.d.ts +11 -0
- package/dist/integration-engine/service/aggregators/TskEventsAggregator.js +49 -0
- package/dist/integration-engine/service/aggregators/TskEventsAggregator.js.map +1 -0
- package/dist/integration-engine/service/aggregators/WazeJamsAggregator.d.ts +10 -0
- package/dist/integration-engine/service/aggregators/WazeJamsAggregator.js +50 -0
- package/dist/integration-engine/service/aggregators/WazeJamsAggregator.js.map +1 -0
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.d.ts +10 -0
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.js +49 -0
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.js.map +1 -0
- package/dist/schema-definitions/helpers/AggregationTaskType.d.ts +7 -0
- package/dist/schema-definitions/helpers/AggregationTaskType.js +12 -0
- package/dist/schema-definitions/helpers/AggregationTaskType.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/FcdEvents.d.ts +14 -0
- package/dist/schema-definitions/sequelize-models/FcdEvents.js +33 -0
- package/dist/schema-definitions/sequelize-models/FcdEvents.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/NdicEvent.d.ts +20 -0
- package/dist/schema-definitions/sequelize-models/NdicEvent.js +44 -0
- package/dist/schema-definitions/sequelize-models/NdicEvent.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/PrahaZasekanaInfo.d.ts +10 -0
- package/dist/schema-definitions/sequelize-models/PrahaZasekanaInfo.js +15 -0
- package/dist/schema-definitions/sequelize-models/PrahaZasekanaInfo.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/TskEvent.d.ts +12 -0
- package/dist/schema-definitions/sequelize-models/TskEvent.js +31 -0
- package/dist/schema-definitions/sequelize-models/TskEvent.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/WazeJams.d.ts +14 -0
- package/dist/schema-definitions/sequelize-models/WazeJams.js +35 -0
- package/dist/schema-definitions/sequelize-models/WazeJams.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/WazeReconstructions.d.ts +13 -0
- package/dist/schema-definitions/sequelize-models/WazeReconstructions.js +33 -0
- package/dist/schema-definitions/sequelize-models/WazeReconstructions.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IFcdEvent.d.ts +8 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IFcdEvent.js +3 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IFcdEvent.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/interfaces/INdicEvent.d.ts +15 -0
- package/dist/schema-definitions/sequelize-models/interfaces/INdicEvent.js +3 -0
- package/dist/schema-definitions/sequelize-models/interfaces/INdicEvent.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/interfaces/ITskEvent.d.ts +7 -0
- package/dist/schema-definitions/sequelize-models/interfaces/ITskEvent.js +3 -0
- package/dist/schema-definitions/sequelize-models/interfaces/ITskEvent.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IWazeJams.d.ts +9 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IWazeJams.js +3 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IWazeJams.js.map +1 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IWazeReconstructions.d.ts +8 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IWazeReconstructions.js +3 -0
- package/dist/schema-definitions/sequelize-models/interfaces/IWazeReconstructions.js.map +1 -0
- package/package.json +63 -0
- package/templates/sqlQueries/FcdEvents.sql +19 -0
- package/templates/sqlQueries/NdicEventsFull.sql +85 -0
- package/templates/sqlQueries/TskEvents.sql +98 -0
- package/templates/sqlQueries/WazeJamsAggregation.sql +27 -0
- package/templates/sqlQueries/WazeReconstructionsAggregation.sql +27 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2019 Operátor ICT, a.s.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
10
|
+
|
|
11
|
+
Source: http://opensource.org/licenses/MIT
|
package/README.md
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[](https://gitlab.com/operator-ict/golemio/code/modules/rush-hour-aggregator/module/commits/master)
|
|
2
|
+
[](https://gitlab.com/operator-ict/golemio/code/modules/rush-hour-aggregator/module/commits/master)
|
|
3
|
+
|
|
4
|
+
# @golemio/rush-hour-aggregation
|
|
5
|
+
|
|
6
|
+
The module aggregates traffic data for Praha Zasekana project. It takes different traffic dataset avaiable in a golemio dataplatform and creates time series data for traffic jams, incidents or reconstructions.
|
|
7
|
+
|
|
8
|
+
[Rabin Praha Zasekana](https://praha-zasekana-backend.rabin.golemio.cz/app)
|
|
9
|
+
[Backend repo Praha Zasekana](https://gitlab.com/operator-ict/golemio/code/praha-zasekana)
|
|
10
|
+
|
|
11
|
+
Developed by http://operatorict.cz
|
|
12
|
+
|
|
13
|
+
# Datasources
|
|
14
|
+
- Waze
|
|
15
|
+
- SDDŘ
|
|
16
|
+
- FCD
|
|
17
|
+
- NDIC
|
|
18
|
+
|
|
19
|
+
# Current setup
|
|
20
|
+
|
|
21
|
+
Every 5 minutes worker starts and for each of the datasource aggregation is run and timeseries data are updated.
|
|
22
|
+
|
|
23
|
+
# Aggregation of older data
|
|
24
|
+
|
|
25
|
+
It is possible to insert or update historic data by creation of manual message in RabbitMQ.
|
|
26
|
+
E.g. to insert data for Fcd for first month in 2022 use following message:
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
{
|
|
30
|
+
"from":"2020-01-01T00:00:00Z",
|
|
31
|
+
"to":"2020-01-31T23:59:59Z",
|
|
32
|
+
"type":"fcd"
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
# Message building blocks
|
|
37
|
+
Message has to be in json format and supports following fields:
|
|
38
|
+
|
|
39
|
+
*from* - Required date time setting when the aggregation has to start. It is formatted in ISO 8601 preferably UTC format with Z at the end
|
|
40
|
+
|
|
41
|
+
*to* - Required date time setting when the aggregation has is ending. It is formatted in ISO 8601 preferably UTC format with Z at the end
|
|
42
|
+
|
|
43
|
+
*type* - Required setting which aggregation should be run. Possible values: WAZEJ, WAZER, SDDR, FCD, NDIC. (Case insensitive)
|
|
44
|
+
|
|
45
|
+
*updateOnDuplicate* - Optional setting for situation when upgrade of already aggregated data is required. Possible values "true", "false". Default value is false. (Note: automatic 5 minute updates have the setting to true)
|
|
46
|
+
|
|
47
|
+
To initiate automatic 5 minute aggregation manually it is possible to send empty message.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dbm;
|
|
4
|
+
var type;
|
|
5
|
+
var seed;
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var Promise;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* We receive the dbmigrate dependency from dbmigrate initially.
|
|
12
|
+
* This enables us to not have to rely on NODE_PATH.
|
|
13
|
+
*/
|
|
14
|
+
exports.setup = function(options, seedLink) {
|
|
15
|
+
dbm = options.dbmigrate;
|
|
16
|
+
type = dbm.dataType;
|
|
17
|
+
seed = seedLink;
|
|
18
|
+
Promise = options.Promise;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
exports.up = function(db) {
|
|
22
|
+
var filePath = path.join(__dirname, 'sqls', '20220405102916-prahaZasekanaAggregation-up.sql');
|
|
23
|
+
return new Promise( function( resolve, reject ) {
|
|
24
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
25
|
+
if (err) return reject(err);
|
|
26
|
+
console.log('received data: ' + data);
|
|
27
|
+
|
|
28
|
+
resolve(data);
|
|
29
|
+
});
|
|
30
|
+
})
|
|
31
|
+
.then(function(data) {
|
|
32
|
+
return db.runSql(data);
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
exports.down = function(db) {
|
|
37
|
+
var filePath = path.join(__dirname, 'sqls', '20220405102916-prahaZasekanaAggregation-down.sql');
|
|
38
|
+
return new Promise( function( resolve, reject ) {
|
|
39
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
40
|
+
if (err) return reject(err);
|
|
41
|
+
console.log('received data: ' + data);
|
|
42
|
+
|
|
43
|
+
resolve(data);
|
|
44
|
+
});
|
|
45
|
+
})
|
|
46
|
+
.then(function(data) {
|
|
47
|
+
return db.runSql(data);
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
exports._meta = {
|
|
52
|
+
"version": 1
|
|
53
|
+
};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/* Replace with your SQL commands */
|
|
2
|
+
|
|
3
|
+
CREATE TABLE waze_jams (
|
|
4
|
+
id varchar(40) NOT NULL,
|
|
5
|
+
measured_at int8 NOT NULL,
|
|
6
|
+
geom_origin geometry NULL,
|
|
7
|
+
geom_startpoint geometry NULL,
|
|
8
|
+
traffic_level text NULL,
|
|
9
|
+
properties json NULL,
|
|
10
|
+
"updated_at" timestamptz NOT NULL,
|
|
11
|
+
"created_at" timestamptz NOT NULL,
|
|
12
|
+
CONSTRAINT waze_jams_pk PRIMARY KEY (id)
|
|
13
|
+
);
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
CREATE TABLE waze_reconstructions (
|
|
17
|
+
id varchar(40) NOT NULL,
|
|
18
|
+
measured_at int8 NOT NULL,
|
|
19
|
+
geom_origin geometry NULL,
|
|
20
|
+
geom_startpoint geometry NULL,
|
|
21
|
+
properties json NULL,
|
|
22
|
+
"updated_at" timestamptz NOT NULL,
|
|
23
|
+
"created_at" timestamptz NOT NULL,
|
|
24
|
+
CONSTRAINT waze_reconstructions_pk PRIMARY KEY (id)
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE TABLE fcd_events (
|
|
28
|
+
source_identification varchar(50) NOT NULL,
|
|
29
|
+
measured_at int8 NOT NULL,
|
|
30
|
+
oriented_route geometry NULL,
|
|
31
|
+
traffic_level varchar(50) NULL,
|
|
32
|
+
properties json NULL,
|
|
33
|
+
"updated_at" timestamptz NOT NULL,
|
|
34
|
+
"created_at" timestamptz NOT NULL,
|
|
35
|
+
CONSTRAINT fcd_events_pk PRIMARY KEY (source_identification, measured_at)
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
CREATE TABLE tsk_std_last_30min (
|
|
39
|
+
measured_at int8 NOT NULL,
|
|
40
|
+
geom geometry NULL,
|
|
41
|
+
sum int8 NULL,
|
|
42
|
+
properties json NULL,
|
|
43
|
+
"updated_at" timestamptz NOT NULL,
|
|
44
|
+
"created_at" timestamptz NOT NULL,
|
|
45
|
+
CONSTRAINT tsk_std_last_30min_pk PRIMARY KEY (measured_at, geom)
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
CREATE TABLE ndic_events_full (
|
|
49
|
+
situation_id varchar(100) NOT NULL,
|
|
50
|
+
situation_record_type varchar(100) NOT NULL,
|
|
51
|
+
situation_version_time timestamptz NOT NULL,
|
|
52
|
+
measured_at int8 NOT NULL,
|
|
53
|
+
geom_origin geometry NULL,
|
|
54
|
+
geom_centroid geometry NULL,
|
|
55
|
+
geom_startpoint geometry NULL,
|
|
56
|
+
geom_endpoint geometry NULL,
|
|
57
|
+
situation_urgency varchar(100) NULL,
|
|
58
|
+
geom_symbol_iconimage text NULL,
|
|
59
|
+
geom_line_pattern text NULL,
|
|
60
|
+
properties json NULL,
|
|
61
|
+
updated_at timestamptz NOT NULL,
|
|
62
|
+
created_at timestamptz NOT NULL,
|
|
63
|
+
CONSTRAINT ndic_events_full_pk PRIMARY KEY (situation_id, situation_record_type, situation_version_time, measured_at)
|
|
64
|
+
);
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * as IntegrationEngine from "./integration-engine/index";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
+
}) : (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
o[k2] = m[k];
|
|
8
|
+
}));
|
|
9
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
+
}) : function(o, v) {
|
|
12
|
+
o["default"] = v;
|
|
13
|
+
});
|
|
14
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
+
if (mod && mod.__esModule) return mod;
|
|
16
|
+
var result = {};
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
+
__setModuleDefault(result, mod);
|
|
19
|
+
return result;
|
|
20
|
+
};
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.IntegrationEngine = void 0;
|
|
23
|
+
// Library exports
|
|
24
|
+
exports.IntegrationEngine = __importStar(require("./integration-engine/index"));
|
|
25
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,kBAAkB;AAClB,gFAA+C"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { BaseWorker } from "@golemio/core/dist/integration-engine";
|
|
2
|
+
import { Message } from "amqplib";
|
|
3
|
+
export declare class AggregationWorker extends BaseWorker {
|
|
4
|
+
static queuePrefix: string;
|
|
5
|
+
static queueName: string;
|
|
6
|
+
static aggregationMethodName: string;
|
|
7
|
+
private aggregationFactory;
|
|
8
|
+
private refreshIntervalInMinutes;
|
|
9
|
+
constructor();
|
|
10
|
+
aggregateData: (message: Message) => Promise<void>;
|
|
11
|
+
private isTaskEmpty;
|
|
12
|
+
private isDateRangeBig;
|
|
13
|
+
private handleEmptyTask;
|
|
14
|
+
private processTask;
|
|
15
|
+
private generateSubTasks;
|
|
16
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.AggregationWorker = void 0;
|
|
16
|
+
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
17
|
+
const config_1 = require("@golemio/core/dist/integration-engine/config");
|
|
18
|
+
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
19
|
+
const errors_1 = require("@golemio/errors");
|
|
20
|
+
const AggregationTaskType_1 = require("../schema-definitions/helpers/AggregationTaskType");
|
|
21
|
+
const AggregationFactory_1 = __importDefault(require("./service/AggregationFactory"));
|
|
22
|
+
const WorkerHelper_1 = __importDefault(require("./WorkerHelper"));
|
|
23
|
+
class AggregationWorker extends integration_engine_1.BaseWorker {
|
|
24
|
+
constructor() {
|
|
25
|
+
super();
|
|
26
|
+
this.refreshIntervalInMinutes = 10;
|
|
27
|
+
this.aggregateData = (message) => __awaiter(this, void 0, void 0, function* () {
|
|
28
|
+
const task = WorkerHelper_1.default.parseMessage(message);
|
|
29
|
+
integration_engine_1.log.debug(`Recieved new task: ${JSON.stringify(task)}`);
|
|
30
|
+
if (this.isTaskEmpty(task)) {
|
|
31
|
+
yield this.handleEmptyTask();
|
|
32
|
+
}
|
|
33
|
+
else if (this.isDateRangeBig(task)) {
|
|
34
|
+
yield this.generateSubTasks(task);
|
|
35
|
+
}
|
|
36
|
+
else if (task.type && task.from && task.to) {
|
|
37
|
+
yield this.processTask(task.type, task.from, task.to, task.updateOnDuplicate);
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
throw new errors_1.CustomError(`Unable to process task: ${JSON.stringify(task)}.`, true, this.constructor.name);
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
this.handleEmptyTask = () => __awaiter(this, void 0, void 0, function* () {
|
|
44
|
+
const to = new Date();
|
|
45
|
+
const from = luxon_1.DateTime.fromJSDate(to).minus({ minutes: this.refreshIntervalInMinutes }).toJSDate();
|
|
46
|
+
for (const type in AggregationTaskType_1.AggregationTaskType) {
|
|
47
|
+
yield this.processTask(type, from, to, true);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
this.processTask = (type, from, to, updateOnDuplicate) => __awaiter(this, void 0, void 0, function* () {
|
|
51
|
+
try {
|
|
52
|
+
const factory = this.aggregationFactory.get(type.toUpperCase());
|
|
53
|
+
yield factory.aggregateAndSave(from, to, updateOnDuplicate);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
integration_engine_1.log.error(`Unable to process task: ${type} from: ${from} to: ${to}. Error stack: ${err.stack}`);
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
this.generateSubTasks = (task) => __awaiter(this, void 0, void 0, function* () {
|
|
60
|
+
const subtasks = WorkerHelper_1.default.generateSubTask(luxon_1.DateTime.fromJSDate(task.from), luxon_1.DateTime.fromJSDate(task.to), this.aggregationFactory.get(task.type).getMaxDuration(), task.type, task.updateOnDuplicate, []);
|
|
61
|
+
for (const subtask of subtasks) {
|
|
62
|
+
yield this.sendMessageToExchange("workers." + AggregationWorker.queuePrefix + "." + AggregationWorker.aggregationMethodName, JSON.stringify(subtask));
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
this.aggregationFactory = new AggregationFactory_1.default();
|
|
66
|
+
}
|
|
67
|
+
isTaskEmpty(task) {
|
|
68
|
+
return !task.from && !task.to && !task.type;
|
|
69
|
+
}
|
|
70
|
+
isDateRangeBig(task) {
|
|
71
|
+
if (task.from && task.to && task.type) {
|
|
72
|
+
const maxDuration = luxon_1.Duration.fromDurationLike(this.aggregationFactory.get(task.type).getMaxDuration());
|
|
73
|
+
const difference = luxon_1.DateTime.fromJSDate(task.to).diff(luxon_1.DateTime.fromJSDate(task.from), "hours");
|
|
74
|
+
return difference && difference > maxDuration;
|
|
75
|
+
}
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
exports.AggregationWorker = AggregationWorker;
|
|
80
|
+
AggregationWorker.queuePrefix = config_1.config.RABBIT_EXCHANGE_NAME + "." + "rush-hour-aggregation";
|
|
81
|
+
AggregationWorker.queueName = "rush-hour-aggregation";
|
|
82
|
+
AggregationWorker.aggregationMethodName = "aggregateData";
|
|
83
|
+
//# sourceMappingURL=AggregationWorker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AggregationWorker.js","sourceRoot":"","sources":["../../src/integration-engine/AggregationWorker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,8EAAwE;AACxE,yEAAsE;AACtE,2DAAqE;AACrE,4CAA8C;AAE9C,2FAAwF;AAExF,sFAA8D;AAC9D,kEAA0C;AAE1C,MAAa,iBAAkB,SAAQ,+BAAU;IAO7C;QACI,KAAK,EAAE,CAAC;QAHJ,6BAAwB,GAAW,EAAE,CAAC;QAOvC,kBAAa,GAAG,CAAO,OAAgB,EAAE,EAAE;YAC9C,MAAM,IAAI,GAAG,sBAAY,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAChD,wBAAG,CAAC,KAAK,CAAC,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE;gBACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;iBAAM,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE;gBAClC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;aACrC;iBAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE;gBAC1C,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;aACjF;iBAAM;gBACH,MAAM,IAAI,oBAAW,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAC1G;QACL,CAAC,CAAA,CAAC;QAiBM,oBAAe,GAAG,GAAwB,EAAE;YAChD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,gBAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClG,KAAK,MAAM,IAAI,IAAI,yCAAmB,EAAE;gBACpC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;aAChD;QACL,CAAC,CAAA,CAAC;QAEM,gBAAW,GAAG,CAAO,IAAY,EAAE,IAAU,EAAE,EAAQ,EAAE,iBAA0B,EAAiB,EAAE;YAC1G,IAAI;gBACA,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAyB,CAAC,CAAC;gBACvF,MAAM,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,EAAE,iBAAiB,CAAC,CAAC;aAC/D;YAAC,OAAO,GAAG,EAAE;gBACV,wBAAG,CAAC,KAAK,CAAC,2BAA2B,IAAI,UAAU,IAAI,QAAQ,EAAE,kBAAkB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;aACnG;QACL,CAAC,CAAA,CAAC;QAEM,qBAAgB,GAAG,CAAO,IAAqB,EAAiB,EAAE;YACtE,MAAM,QAAQ,GAAG,sBAAY,CAAC,eAAe,CACzC,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAK,CAAC,EAC/B,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAG,CAAC,EAC7B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAK,CAAC,CAAC,cAAc,EAAE,EACxD,IAAI,CAAC,IAAK,EACV,IAAI,CAAC,iBAAiB,EACtB,EAAE,CACL,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC5B,MAAM,IAAI,CAAC,qBAAqB,CAC5B,UAAU,GAAG,iBAAiB,CAAC,WAAW,GAAG,GAAG,GAAG,iBAAiB,CAAC,qBAAqB,EAC1F,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAC1B,CAAC;aACL;QACL,CAAC,CAAA,CAAC;QAjEE,IAAI,CAAC,kBAAkB,GAAG,IAAI,4BAAkB,EAAE,CAAC;IACvD,CAAC;IAgBO,WAAW,CAAC,IAAqB;QACrC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAChD,CAAC;IAEO,cAAc,CAAC,IAAqB;QACxC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE;YACnC,MAAM,WAAW,GAAG,gBAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;YACvG,MAAM,UAAU,GAAG,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAE9F,OAAO,UAAU,IAAI,UAAU,GAAG,WAAW,CAAC;SACjD;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;;AAvCL,8CA2EC;AA1EiB,6BAAW,GAAG,eAAM,CAAC,oBAAoB,GAAG,GAAG,GAAG,uBAAuB,CAAC;AAC1E,2BAAS,GAAG,uBAAuB,CAAC;AACpC,uCAAqB,GAAG,eAAe,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { AggregationTaskType } from "../schema-definitions/helpers/AggregationTaskType";
|
|
2
|
+
import { DateTime, DurationLike } from "@golemio/core/dist/shared/luxon";
|
|
3
|
+
import { Message } from "amqplib";
|
|
4
|
+
import AggregationTask from "./business/AggregationTask";
|
|
5
|
+
export default class WorkerHelper {
|
|
6
|
+
static parseMessage(message: Message): AggregationTask;
|
|
7
|
+
static generateSubTask: (tmpFrom: DateTime, to: DateTime, maxDuration: DurationLike, taskType: AggregationTaskType, updateOnDuplicate: boolean, result: AggregationTask[]) => AggregationTask[];
|
|
8
|
+
private static parseContent;
|
|
9
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
var _a;
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
const AggregationTaskType_1 = require("../schema-definitions/helpers/AggregationTaskType");
|
|
8
|
+
const helpers_1 = require("@golemio/core/dist/integration-engine/helpers");
|
|
9
|
+
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
10
|
+
const errors_1 = require("@golemio/errors");
|
|
11
|
+
const AggregationTask_1 = __importDefault(require("./business/AggregationTask"));
|
|
12
|
+
class WorkerHelper {
|
|
13
|
+
static parseMessage(message) {
|
|
14
|
+
var _b;
|
|
15
|
+
const task = WorkerHelper.parseContent(message.content.toString());
|
|
16
|
+
if (!task)
|
|
17
|
+
return {};
|
|
18
|
+
const type = AggregationTaskType_1.AggregationTaskType[(_b = task.type) === null || _b === void 0 ? void 0 : _b.toUpperCase()];
|
|
19
|
+
if (!type)
|
|
20
|
+
throw new errors_1.CustomError(`Unknown aggregation task type ${JSON.stringify(task)}`, true, this.constructor.name);
|
|
21
|
+
if (!task.from || !task.to)
|
|
22
|
+
throw new errors_1.CustomError(`Unsupported aggregation task ${JSON.stringify(task)}`, true, this.constructor.name);
|
|
23
|
+
const result = new AggregationTask_1.default(luxon_1.DateTime.fromISO(task.from).toJSDate(), luxon_1.DateTime.fromISO(task.to).toJSDate(), type, task.updateOnDuplicate ? task.updateOnDuplicate.toString().toLowerCase() == "true" : undefined);
|
|
24
|
+
if (result.from > result.to)
|
|
25
|
+
throw new errors_1.CustomError(`Unsupported task date range ${JSON.stringify(task)}`, true, this.constructor.name);
|
|
26
|
+
return result;
|
|
27
|
+
}
|
|
28
|
+
static parseContent(messageContent) {
|
|
29
|
+
try {
|
|
30
|
+
return messageContent.length > 0 ? JSON.parse(messageContent) : undefined;
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
throw new errors_1.CustomError(`Unable to parse json from message: ${messageContent}`, true, this.constructor.name, undefined, err);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
exports.default = WorkerHelper;
|
|
38
|
+
_a = WorkerHelper;
|
|
39
|
+
WorkerHelper.generateSubTask = (tmpFrom, to, maxDuration, taskType, updateOnDuplicate, result) => {
|
|
40
|
+
const tmpTo = tmpFrom.plus(maxDuration);
|
|
41
|
+
if (tmpTo >= to) {
|
|
42
|
+
helpers_1.log.silly(`generating task from:${tmpFrom.toISO()}, to:${to.toISO()}`);
|
|
43
|
+
result.push(new AggregationTask_1.default(tmpFrom.toJSDate(), to.toJSDate(), taskType, updateOnDuplicate));
|
|
44
|
+
return result;
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
helpers_1.log.silly(`generating task from:${tmpFrom.toISO()}, to:${tmpTo.toISO()}`);
|
|
48
|
+
result.push(new AggregationTask_1.default(tmpFrom.toJSDate(), tmpTo.toJSDate(), taskType, updateOnDuplicate));
|
|
49
|
+
return _a.generateSubTask(tmpTo, to, maxDuration, taskType, updateOnDuplicate, result);
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
//# sourceMappingURL=WorkerHelper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WorkerHelper.js","sourceRoot":"","sources":["../../src/integration-engine/WorkerHelper.ts"],"names":[],"mappings":";;;;;;AAAA,2FAAuE;AACvE,2EAAoE;AACpE,2DAAyE;AACzE,4CAA8C;AAE9C,iFAAyD;AAEzD,MAAqB,YAAY;IACtB,MAAM,CAAC,YAAY,CAAC,OAAgB;;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC,IAAI;YAAE,OAAO,EAAqB,CAAC;QAExC,MAAM,IAAI,GAAS,yCAAoB,CAAC,MAAA,IAAI,CAAC,IAAI,0CAAE,WAAW,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,oBAAW,CAAC,iCAAiC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACvH,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE;YACtB,MAAM,IAAI,oBAAW,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAE/G,MAAM,MAAM,GAAG,IAAI,yBAAe,CAC9B,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EACtC,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,EACpC,IAAI,EACJ,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,SAAS,CACjG,CAAC;QAEF,IAAI,MAAM,CAAC,IAAK,GAAG,MAAM,CAAC,EAAG;YACzB,MAAM,IAAI,oBAAW,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAE9G,OAAO,MAAM,CAAC;IAClB,CAAC;IAwBO,MAAM,CAAC,YAAY,CAAC,cAAsB;QAC9C,IAAI;YACA,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;SAC7E;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,oBAAW,CACjB,sCAAsC,cAAc,EAAE,EACtD,IAAI,EACJ,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,SAAS,EACT,GAAG,CACN,CAAC;SACL;IACL,CAAC;;AA1DL,+BA2DC;;AAnCiB,4BAAe,GAAG,CAC5B,OAAiB,EACjB,EAAY,EACZ,WAAyB,EACzB,QAA6B,EAC7B,iBAA0B,EAC1B,MAAyB,EACR,EAAE;IACnB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxC,IAAI,KAAK,IAAI,EAAE,EAAE;QACb,aAAG,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC,IAAI,yBAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAEjG,OAAO,MAAM,CAAC;KACjB;SAAM;QACH,aAAG,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,KAAK,EAAE,QAAQ,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,IAAI,CAAC,IAAI,yBAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAEpG,OAAO,EAAI,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;KAC5F;AACL,CAAE,CAAA"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { AggregationTaskType } from "../../schema-definitions/helpers/AggregationTaskType";
|
|
2
|
+
import IAggregationTask from "./interfaces/IAggregationTask";
|
|
3
|
+
export default class AggregationTask implements IAggregationTask {
|
|
4
|
+
from?: Date;
|
|
5
|
+
to?: Date;
|
|
6
|
+
type?: AggregationTaskType;
|
|
7
|
+
updateOnDuplicate: boolean;
|
|
8
|
+
get fromUnixTimestamp(): Number | undefined;
|
|
9
|
+
get toUnixTimestamp(): Number | undefined;
|
|
10
|
+
constructor(from: Date, to: Date, type: AggregationTaskType, updateOnDuplicate?: boolean);
|
|
11
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
class AggregationTask {
|
|
4
|
+
constructor(from, to, type, updateOnDuplicate) {
|
|
5
|
+
this.from = from;
|
|
6
|
+
this.to = to;
|
|
7
|
+
this.type = type;
|
|
8
|
+
this.updateOnDuplicate = updateOnDuplicate || false;
|
|
9
|
+
}
|
|
10
|
+
get fromUnixTimestamp() {
|
|
11
|
+
return this.from ? this.from.getTime() : undefined;
|
|
12
|
+
}
|
|
13
|
+
get toUnixTimestamp() {
|
|
14
|
+
return this.to ? this.to.getTime() : undefined;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
exports.default = AggregationTask;
|
|
18
|
+
//# sourceMappingURL=AggregationTask.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AggregationTask.js","sourceRoot":"","sources":["../../../src/integration-engine/business/AggregationTask.ts"],"names":[],"mappings":";;AAGA,MAAqB,eAAe;IAchC,YAAY,IAAU,EAAE,EAAQ,EAAE,IAAyB,EAAE,iBAA2B;QACpF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,IAAI,KAAK,CAAC;IACxD,CAAC;IAbD,IAAW,iBAAiB;QACxB,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACvD,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;CAQJ;AApBD,kCAoBC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IAggregationTask.js","sourceRoot":"","sources":["../../../../src/integration-engine/business/interfaces/IAggregationTask.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AggregationTaskType } from "../../schema-definitions/helpers/AggregationTaskType";
|
|
2
|
+
import { BindOrReplacements } from "@golemio/core/dist/shared/sequelize";
|
|
3
|
+
export default class AggregationRepository {
|
|
4
|
+
private static instance;
|
|
5
|
+
private sqlQueriesFileNames;
|
|
6
|
+
private sqlQueries;
|
|
7
|
+
private relativeModuleRootPath;
|
|
8
|
+
private sqlQueriesFolder;
|
|
9
|
+
static getInstance(): AggregationRepository;
|
|
10
|
+
private constructor();
|
|
11
|
+
aggregationQuery: <T extends object>(type: AggregationTaskType, bindings?: BindOrReplacements | undefined, replacements?: BindOrReplacements | undefined) => Promise<T[]>;
|
|
12
|
+
protected loadSqlQuery: (type: AggregationTaskType) => string;
|
|
13
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const AggregationTaskType_1 = require("../../schema-definitions/helpers/AggregationTaskType");
|
|
16
|
+
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
17
|
+
const sequelize_1 = require("@golemio/core/dist/shared/sequelize");
|
|
18
|
+
const errors_1 = require("@golemio/errors");
|
|
19
|
+
const fs_1 = __importDefault(require("fs"));
|
|
20
|
+
const path_1 = __importDefault(require("path"));
|
|
21
|
+
class AggregationRepository {
|
|
22
|
+
constructor() {
|
|
23
|
+
this.relativeModuleRootPath = "../../../"; // <= src/#ie/dataAccess
|
|
24
|
+
this.sqlQueriesFolder = "templates/sqlQueries/";
|
|
25
|
+
this.aggregationQuery = (type, bindings, replacements) => __awaiter(this, void 0, void 0, function* () {
|
|
26
|
+
return yield integration_engine_1.PostgresConnector.getConnection().query(this.loadSqlQuery(type), {
|
|
27
|
+
type: sequelize_1.QueryTypes.SELECT,
|
|
28
|
+
raw: true,
|
|
29
|
+
bind: bindings,
|
|
30
|
+
replacements: replacements,
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
this.loadSqlQuery = (type) => {
|
|
34
|
+
if (!this.sqlQueries.get(type)) {
|
|
35
|
+
const queryFileName = this.sqlQueriesFileNames.get(type);
|
|
36
|
+
if (!queryFileName)
|
|
37
|
+
throw new errors_1.CustomError(`Couldn't find sql query for type: ${type}`, false, this.constructor.name);
|
|
38
|
+
const pathToQuery = path_1.default.resolve(__dirname, this.relativeModuleRootPath, this.sqlQueriesFolder, queryFileName);
|
|
39
|
+
this.sqlQueries.set(type, fs_1.default.readFileSync(pathToQuery).toString());
|
|
40
|
+
}
|
|
41
|
+
return this.sqlQueries.get(type);
|
|
42
|
+
};
|
|
43
|
+
this.sqlQueriesFileNames = new Map([
|
|
44
|
+
[AggregationTaskType_1.AggregationTaskType.FCD, "FcdEvents.sql"],
|
|
45
|
+
[AggregationTaskType_1.AggregationTaskType.NDIC, "NdicEventsFull.sql"],
|
|
46
|
+
[AggregationTaskType_1.AggregationTaskType.SDDR, "TskEvents.sql"],
|
|
47
|
+
[AggregationTaskType_1.AggregationTaskType.WAZEJ, "WazeJamsAggregation.sql"],
|
|
48
|
+
[AggregationTaskType_1.AggregationTaskType.WAZER, "WazeReconstructionsAggregation.sql"],
|
|
49
|
+
]);
|
|
50
|
+
this.sqlQueries = new Map();
|
|
51
|
+
}
|
|
52
|
+
static getInstance() {
|
|
53
|
+
if (!AggregationRepository.instance) {
|
|
54
|
+
AggregationRepository.instance = new AggregationRepository();
|
|
55
|
+
}
|
|
56
|
+
return AggregationRepository.instance;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
exports.default = AggregationRepository;
|
|
60
|
+
//# sourceMappingURL=AggregationRepository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AggregationRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/AggregationRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8FAAuE;AACvE,8EAA0E;AAC1E,mEAAqF;AACrF,4CAA8C;AAC9C,4CAAoB;AACpB,gDAAwB;AAExB,MAAqB,qBAAqB;IAetC;QAXQ,2BAAsB,GAAW,WAAW,CAAC,CAAC,wBAAwB;QACtE,qBAAgB,GAAW,uBAAuB,CAAC;QAqBpD,qBAAgB,GAAG,CACtB,IAAyB,EACzB,QAA6B,EAC7B,YAAiC,EACrB,EAAE;YACd,OAAO,MAAM,sCAAiB,CAAC,aAAa,EAAE,CAAC,KAAK,CAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE;gBAC7E,IAAI,EAAE,sBAAU,CAAC,MAAM;gBACvB,GAAG,EAAE,IAAI;gBACT,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,YAAY;aAC7B,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;QAEQ,iBAAY,GAAG,CAAC,IAAyB,EAAU,EAAE;YAC3D,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBAC5B,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzD,IAAI,CAAC,aAAa;oBAAE,MAAM,IAAI,oBAAW,CAAC,qCAAqC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAErH,MAAM,WAAW,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,sBAAsB,EAAE,IAAI,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;gBAC/G,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,YAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;aACtE;YAED,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QACtC,CAAC,CAAC;QAjCE,IAAI,CAAC,mBAAmB,GAAG,IAAI,GAAG,CAA8B;YAC5D,CAAC,yCAAmB,CAAC,GAAG,EAAE,eAAe,CAAC;YAC1C,CAAC,yCAAmB,CAAC,IAAI,EAAE,oBAAoB,CAAC;YAChD,CAAC,yCAAmB,CAAC,IAAI,EAAE,eAAe,CAAC;YAC3C,CAAC,yCAAmB,CAAC,KAAK,EAAE,yBAAyB,CAAC;YACtD,CAAC,yCAAmB,CAAC,KAAK,EAAE,oCAAoC,CAAC;SACpE,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAA+B,CAAC;IAC7D,CAAC;IAjBM,MAAM,CAAC,WAAW;QACrB,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE;YACjC,qBAAqB,CAAC,QAAQ,GAAG,IAAI,qBAAqB,EAAE,CAAC;SAChE;QAED,OAAO,qBAAqB,CAAC,QAAQ,CAAC;IAC1C,CAAC;CAqCJ;AAlDD,wCAkDC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import FcdEvent from "../../schema-definitions/sequelize-models/FcdEvents";
|
|
2
|
+
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine";
|
|
3
|
+
export default class FcdEventsRepository extends PostgresModel implements IModel {
|
|
4
|
+
constructor();
|
|
5
|
+
saveData: (data: FcdEvent[], updateOnDuplicate: boolean) => Promise<void>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const AggregationTaskType_1 = require("../../schema-definitions/helpers/AggregationTaskType");
|
|
16
|
+
const FcdEvents_1 = __importDefault(require("../../schema-definitions/sequelize-models/FcdEvents"));
|
|
17
|
+
const PrahaZasekanaInfo_1 = __importDefault(require("../../schema-definitions/sequelize-models/PrahaZasekanaInfo"));
|
|
18
|
+
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
19
|
+
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
20
|
+
class FcdEventsRepository extends integration_engine_1.PostgresModel {
|
|
21
|
+
constructor() {
|
|
22
|
+
super("FcdEventsRepository", {
|
|
23
|
+
outputSequelizeAttributes: FcdEvents_1.default.attributeModel,
|
|
24
|
+
pgTableName: PrahaZasekanaInfo_1.default.TABLE_NAMES[AggregationTaskType_1.AggregationTaskType.FCD],
|
|
25
|
+
pgSchema: PrahaZasekanaInfo_1.default.SCHEMA,
|
|
26
|
+
savingType: "insertOnly",
|
|
27
|
+
}, new golemio_validator_1.JSONSchemaValidator("FcdEventsRepository", FcdEvents_1.default.arrayJsonSchema, true));
|
|
28
|
+
this.saveData = (data, updateOnDuplicate) => __awaiter(this, void 0, void 0, function* () {
|
|
29
|
+
yield this["sequelizeModel"].bulkCreate(data, {
|
|
30
|
+
ignoreDuplicates: !updateOnDuplicate,
|
|
31
|
+
returning: false,
|
|
32
|
+
updateOnDuplicate: updateOnDuplicate ? ["oriented_route", "traffic_level", "properties"] : undefined,
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
exports.default = FcdEventsRepository;
|
|
38
|
+
//# sourceMappingURL=FcdEventsRepository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FcdEventsRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/FcdEventsRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8FAAuE;AACvE,oGAAuD;AACvD,oHAAwE;AACxE,8EAA8E;AAC9E,mFAAkF;AAElF,MAAqB,mBAAoB,SAAQ,kCAAa;IAC1D;QACI,KAAK,CACD,qBAAqB,EACrB;YACI,yBAAyB,EAAE,mBAAQ,CAAC,cAAc;YAClD,WAAW,EAAE,2BAAiB,CAAC,WAAW,CAAC,yCAAmB,CAAC,GAAG,CAAC;YACnE,QAAQ,EAAE,2BAAiB,CAAC,MAAM;YAClC,UAAU,EAAE,YAAY;SAC3B,EACD,IAAI,uCAAmB,CAAC,qBAAqB,EAAE,mBAAQ,CAAC,eAAe,EAAE,IAAI,CAAC,CACjF,CAAC;QAGC,aAAQ,GAAG,CAAO,IAAgB,EAAE,iBAA0B,EAAE,EAAE;YACrE,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,UAAU,CAAW,IAAI,EAAE;gBACpD,gBAAgB,EAAE,CAAC,iBAAiB;gBACpC,SAAS,EAAE,KAAK;gBAChB,iBAAiB,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,gBAAgB,EAAE,eAAe,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;aACvG,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;IARF,CAAC;CASJ;AArBD,sCAqBC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import NdicEvent from "../../schema-definitions/sequelize-models/NdicEvent";
|
|
2
|
+
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine";
|
|
3
|
+
export default class NdicEventsRepository extends PostgresModel implements IModel {
|
|
4
|
+
constructor();
|
|
5
|
+
saveData: (data: NdicEvent[], updateOnDuplicate: boolean) => Promise<void>;
|
|
6
|
+
}
|