@golemio/rush-hour-aggregation 1.5.1 → 1.5.2-dev.2941756587
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/db/migrations/postgresql/20261004194210-snapshots-latest-index-usage.js +53 -0
- package/db/migrations/postgresql/sqls/20261004194210-snapshots-latest-index-usage-down.sql +203 -0
- package/db/migrations/postgresql/sqls/20261004194210-snapshots-latest-index-usage-up.sql +203 -0
- package/dist/integration-engine/WorkerHelper.d.ts +0 -3
- package/dist/integration-engine/WorkerHelper.js +0 -27
- package/dist/integration-engine/WorkerHelper.js.map +1 -1
- package/dist/integration-engine/constants.d.ts +1 -0
- package/dist/integration-engine/constants.js +5 -0
- package/dist/integration-engine/constants.js.map +1 -0
- package/dist/integration-engine/dataAccess/AbstractEventsRepository.d.ts +7 -0
- package/dist/integration-engine/dataAccess/AbstractEventsRepository.js +11 -8
- package/dist/integration-engine/dataAccess/AbstractEventsRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/AggregationRepository.d.ts +5 -4
- package/dist/integration-engine/dataAccess/AggregationRepository.js +25 -12
- package/dist/integration-engine/dataAccess/AggregationRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/DataRetentionRepository.d.ts +2 -1
- package/dist/integration-engine/dataAccess/DataRetentionRepository.js +22 -2
- package/dist/integration-engine/dataAccess/DataRetentionRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/FcdEventsRepository.d.ts +3 -1
- package/dist/integration-engine/dataAccess/FcdEventsRepository.js +2 -2
- package/dist/integration-engine/dataAccess/FcdEventsRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/NdicEventRepository.d.ts +3 -1
- package/dist/integration-engine/dataAccess/NdicEventRepository.js +2 -2
- package/dist/integration-engine/dataAccess/NdicEventRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/TskEventsRepository.d.ts +3 -1
- package/dist/integration-engine/dataAccess/TskEventsRepository.js +2 -2
- package/dist/integration-engine/dataAccess/TskEventsRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/WazeJamsRepository.d.ts +3 -1
- package/dist/integration-engine/dataAccess/WazeJamsRepository.js +2 -2
- package/dist/integration-engine/dataAccess/WazeJamsRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/WazeReconstructionsRepository.d.ts +3 -1
- package/dist/integration-engine/dataAccess/WazeReconstructionsRepository.js +2 -2
- package/dist/integration-engine/dataAccess/WazeReconstructionsRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/interfaces/IAggregationRepository.d.ts +5 -0
- package/dist/integration-engine/dataAccess/interfaces/IAggregationRepository.js +3 -0
- package/dist/integration-engine/dataAccess/interfaces/IAggregationRepository.js.map +1 -0
- package/dist/integration-engine/dataAccess/interfaces/IDataRetentionRepository.d.ts +3 -0
- package/dist/integration-engine/dataAccess/interfaces/IDataRetentionRepository.js +3 -0
- package/dist/integration-engine/dataAccess/interfaces/IDataRetentionRepository.js.map +1 -0
- package/dist/integration-engine/index.d.ts +2 -2
- package/dist/integration-engine/index.js +3 -17
- package/dist/integration-engine/index.js.map +1 -1
- package/dist/integration-engine/ioc/Di.d.ts +3 -0
- package/dist/integration-engine/ioc/Di.js +20 -0
- package/dist/integration-engine/ioc/Di.js.map +1 -0
- package/dist/integration-engine/ioc/RushHourAggregationContainerToken.d.ts +8 -0
- package/dist/integration-engine/ioc/RushHourAggregationContainerToken.js +12 -0
- package/dist/integration-engine/ioc/RushHourAggregationContainerToken.js.map +1 -0
- package/dist/integration-engine/service/AggregationFactory.d.ts +6 -2
- package/dist/integration-engine/service/AggregationFactory.js +32 -8
- package/dist/integration-engine/service/AggregationFactory.js.map +1 -1
- package/dist/integration-engine/service/aggregators/AbstractAggregator.d.ts +5 -0
- package/dist/integration-engine/service/aggregators/AbstractAggregator.js +6 -3
- package/dist/integration-engine/service/aggregators/AbstractAggregator.js.map +1 -1
- package/dist/integration-engine/service/aggregators/FcdEventsAggregator.d.ts +4 -1
- package/dist/integration-engine/service/aggregators/FcdEventsAggregator.js +4 -5
- package/dist/integration-engine/service/aggregators/FcdEventsAggregator.js.map +1 -1
- package/dist/integration-engine/service/aggregators/NdicEventsAggregator.d.ts +4 -1
- package/dist/integration-engine/service/aggregators/NdicEventsAggregator.js +4 -5
- package/dist/integration-engine/service/aggregators/NdicEventsAggregator.js.map +1 -1
- package/dist/integration-engine/service/aggregators/TskEventsAggregator.d.ts +4 -1
- package/dist/integration-engine/service/aggregators/TskEventsAggregator.js +4 -5
- package/dist/integration-engine/service/aggregators/TskEventsAggregator.js.map +1 -1
- package/dist/integration-engine/service/aggregators/WazeJamsAggregator.d.ts +4 -1
- package/dist/integration-engine/service/aggregators/WazeJamsAggregator.js +4 -5
- package/dist/integration-engine/service/aggregators/WazeJamsAggregator.js.map +1 -1
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.d.ts +4 -1
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.js +7 -5
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.js.map +1 -1
- package/dist/integration-engine/service/interfaces/IAggregationFactory.d.ts +5 -0
- package/dist/integration-engine/service/interfaces/IAggregationFactory.js +3 -0
- package/dist/integration-engine/service/interfaces/IAggregationFactory.js.map +1 -0
- package/dist/integration-engine/workers/RushHourAggregationWorker.d.ts +7 -0
- package/dist/integration-engine/workers/RushHourAggregationWorker.js +21 -0
- package/dist/integration-engine/workers/RushHourAggregationWorker.js.map +1 -0
- package/dist/integration-engine/workers/interfaces/IAggregateDataTaskInput.d.ts +8 -0
- package/dist/integration-engine/workers/interfaces/IAggregateDataTaskInput.js +3 -0
- package/dist/integration-engine/workers/interfaces/IAggregateDataTaskInput.js.map +1 -0
- package/dist/integration-engine/workers/schema/AggregateDataTaskInputSchema.d.ts +9 -0
- package/dist/integration-engine/workers/schema/AggregateDataTaskInputSchema.js +49 -0
- package/dist/integration-engine/workers/schema/AggregateDataTaskInputSchema.js.map +1 -0
- package/dist/integration-engine/workers/task/AggregateDataTask.d.ts +24 -0
- package/dist/integration-engine/workers/task/AggregateDataTask.js +118 -0
- package/dist/integration-engine/workers/task/AggregateDataTask.js.map +1 -0
- package/dist/integration-engine/workers/task/RefreshHistoricViewsTask.d.ts +11 -0
- package/dist/integration-engine/workers/task/RefreshHistoricViewsTask.js +58 -0
- package/dist/integration-engine/workers/task/RefreshHistoricViewsTask.js.map +1 -0
- package/docs/asyncapi.yaml +1 -1
- package/docs/implementation_documentation.md +17 -11
- package/package.json +1 -1
- package/templates/sqlQueries/NdicEventsFull.sql +3 -0
- package/dist/integration-engine/AggregationWorker.d.ts +0 -18
- package/dist/integration-engine/AggregationWorker.js +0 -88
- package/dist/integration-engine/AggregationWorker.js.map +0 -1
- package/dist/integration-engine/queueDefinitions.d.ts +0 -3
- package/dist/integration-engine/queueDefinitions.js +0 -36
- package/dist/integration-engine/queueDefinitions.js.map +0 -1
- package/docs/keeping_a_changelog.md +0 -48
package/README.md
CHANGED
|
@@ -57,7 +57,7 @@ This module aggregates traffic data for Praha dopravní project. It takes differ
|
|
|
57
57
|
|
|
58
58
|
## Current setup
|
|
59
59
|
|
|
60
|
-
Every 5 minutes worker starts and for each
|
|
60
|
+
Every 5 minutes worker starts and for each datasource except SDDR aggregation is run and timeseries data are updated.
|
|
61
61
|
|
|
62
62
|
## Aggregation of older data
|
|
63
63
|
|
|
@@ -88,4 +88,4 @@ _updateOnDuplicate_ - Optional setting for situation when upgrade of already agg
|
|
|
88
88
|
|
|
89
89
|
_refreshView_ - Optional setting to refresh latest materialized view. Possible values "true", "false". Default value is false. (Note: automatic 5 minute updates have the setting to true)
|
|
90
90
|
|
|
91
|
-
To initiate automatic 5 minute aggregation manually it is possible to send empty message.
|
|
91
|
+
To initiate automatic 5 minute aggregation manually it is possible to send empty message. The empty message sends a separate message for each of the types FCD, NDIC, WAZEJ and WAZER. SDDR is not part of the automatic aggregation, because its source no longer receives data; send a message with type SDDR to aggregate it.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dbm;
|
|
4
|
+
var type;
|
|
5
|
+
var seed;
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var Promise;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* We receive the dbmigrate dependency from dbmigrate initially.
|
|
12
|
+
* This enables us to not have to rely on NODE_PATH.
|
|
13
|
+
*/
|
|
14
|
+
exports.setup = function(options, seedLink) {
|
|
15
|
+
dbm = options.dbmigrate;
|
|
16
|
+
type = dbm.dataType;
|
|
17
|
+
seed = seedLink;
|
|
18
|
+
Promise = options.Promise;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
exports.up = function(db) {
|
|
22
|
+
var filePath = path.join(__dirname, 'sqls', '20261004194210-snapshots-latest-index-usage-up.sql');
|
|
23
|
+
return new Promise( function( resolve, reject ) {
|
|
24
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
25
|
+
if (err) return reject(err);
|
|
26
|
+
console.log('received data: ' + data);
|
|
27
|
+
|
|
28
|
+
resolve(data);
|
|
29
|
+
});
|
|
30
|
+
})
|
|
31
|
+
.then(function(data) {
|
|
32
|
+
return db.runSql(data);
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
exports.down = function(db) {
|
|
37
|
+
var filePath = path.join(__dirname, 'sqls', '20261004194210-snapshots-latest-index-usage-down.sql');
|
|
38
|
+
return new Promise( function( resolve, reject ) {
|
|
39
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
40
|
+
if (err) return reject(err);
|
|
41
|
+
console.log('received data: ' + data);
|
|
42
|
+
|
|
43
|
+
resolve(data);
|
|
44
|
+
});
|
|
45
|
+
})
|
|
46
|
+
.then(function(data) {
|
|
47
|
+
return db.runSql(data);
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
exports._meta = {
|
|
52
|
+
"version": 1
|
|
53
|
+
};
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
DROP MATERIALIZED VIEW ndic_events_full_snapshots_latest;
|
|
2
|
+
|
|
3
|
+
CREATE MATERIALIZED VIEW ndic_events_full_snapshots_latest AS WITH config AS (
|
|
4
|
+
SELECT 1800000 AS max_period,
|
|
5
|
+
date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000::double precision AS min_measured_at
|
|
6
|
+
)
|
|
7
|
+
SELECT s.start_measured_at,
|
|
8
|
+
s.end_measured_at,
|
|
9
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
10
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
11
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
12
|
+
CASE
|
|
13
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
14
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
15
|
+
END + (( SELECT config.max_period
|
|
16
|
+
FROM config)) AS end_measured_at,
|
|
17
|
+
s_1.is_start,
|
|
18
|
+
s_1.is_end
|
|
19
|
+
FROM ( SELECT s_2.measured_at,
|
|
20
|
+
s_2.prev_measured_at,
|
|
21
|
+
s_2.next_measured_at,
|
|
22
|
+
s_2."?column?",
|
|
23
|
+
s_2."?column?_1" AS "?column?",
|
|
24
|
+
s_2.is_start,
|
|
25
|
+
s_2.is_end
|
|
26
|
+
FROM ( SELECT s_3.measured_at,
|
|
27
|
+
s_3.prev_measured_at,
|
|
28
|
+
s_3.next_measured_at,
|
|
29
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
30
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
31
|
+
CASE
|
|
32
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
33
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
34
|
+
ELSE false
|
|
35
|
+
END AS is_start,
|
|
36
|
+
CASE
|
|
37
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
38
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
39
|
+
ELSE false
|
|
40
|
+
END AS is_end
|
|
41
|
+
FROM ( SELECT s_4.measured_at,
|
|
42
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
43
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
44
|
+
FROM ( SELECT DISTINCT ndic_events_full.measured_at
|
|
45
|
+
FROM ndic_events_full WHERE ndic_events_full.measured_at::double precision >= (( SELECT config.min_measured_at
|
|
46
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
47
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
48
|
+
WHERE s.is_start IS TRUE;
|
|
49
|
+
|
|
50
|
+
CREATE UNIQUE INDEX ndic_events_full_snapshots_latest_start_measured_at_idx ON ndic_events_full_snapshots_latest(start_measured_at int8_ops);
|
|
51
|
+
|
|
52
|
+
DROP MATERIALIZED VIEW tsk_std_last_30min_snapshots_latest;
|
|
53
|
+
|
|
54
|
+
CREATE MATERIALIZED VIEW tsk_std_last_30min_snapshots_latest AS WITH config AS (
|
|
55
|
+
SELECT 300000 AS max_period,
|
|
56
|
+
date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000::double precision AS min_measured_at
|
|
57
|
+
)
|
|
58
|
+
SELECT s.start_measured_at,
|
|
59
|
+
s.end_measured_at,
|
|
60
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
61
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
62
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
63
|
+
CASE
|
|
64
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
65
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
66
|
+
END + (( SELECT config.max_period
|
|
67
|
+
FROM config)) AS end_measured_at,
|
|
68
|
+
s_1.is_start,
|
|
69
|
+
s_1.is_end
|
|
70
|
+
FROM ( SELECT s_2.measured_at,
|
|
71
|
+
s_2.prev_measured_at,
|
|
72
|
+
s_2.next_measured_at,
|
|
73
|
+
s_2."?column?",
|
|
74
|
+
s_2."?column?_1" AS "?column?",
|
|
75
|
+
s_2.is_start,
|
|
76
|
+
s_2.is_end
|
|
77
|
+
FROM ( SELECT s_3.measured_at,
|
|
78
|
+
s_3.prev_measured_at,
|
|
79
|
+
s_3.next_measured_at,
|
|
80
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
81
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
82
|
+
CASE
|
|
83
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
84
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
85
|
+
ELSE false
|
|
86
|
+
END AS is_start,
|
|
87
|
+
CASE
|
|
88
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
89
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
90
|
+
ELSE false
|
|
91
|
+
END AS is_end
|
|
92
|
+
FROM ( SELECT s_4.measured_at,
|
|
93
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
94
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
95
|
+
FROM ( SELECT DISTINCT tsk_std_last_30min.measured_at
|
|
96
|
+
FROM tsk_std_last_30min WHERE tsk_std_last_30min.measured_at::double precision >= (( SELECT config.min_measured_at
|
|
97
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
98
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
99
|
+
WHERE s.is_start IS TRUE;
|
|
100
|
+
|
|
101
|
+
CREATE UNIQUE INDEX tsk_std_last_30min_snapshots_latest_start_measured_at_idx ON tsk_std_last_30min_snapshots_latest(start_measured_at int8_ops);
|
|
102
|
+
|
|
103
|
+
DROP MATERIALIZED VIEW waze_jams_snapshots_latest;
|
|
104
|
+
|
|
105
|
+
CREATE MATERIALIZED VIEW waze_jams_snapshots_latest AS WITH config AS (
|
|
106
|
+
SELECT 120000 AS max_period,
|
|
107
|
+
date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000::double precision AS min_measured_at
|
|
108
|
+
)
|
|
109
|
+
SELECT s.start_measured_at,
|
|
110
|
+
s.end_measured_at,
|
|
111
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
112
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
113
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
114
|
+
CASE
|
|
115
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
116
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
117
|
+
END + (( SELECT config.max_period
|
|
118
|
+
FROM config)) AS end_measured_at,
|
|
119
|
+
s_1.is_start,
|
|
120
|
+
s_1.is_end
|
|
121
|
+
FROM ( SELECT s_2.measured_at,
|
|
122
|
+
s_2.prev_measured_at,
|
|
123
|
+
s_2.next_measured_at,
|
|
124
|
+
s_2."?column?",
|
|
125
|
+
s_2."?column?_1" AS "?column?",
|
|
126
|
+
s_2.is_start,
|
|
127
|
+
s_2.is_end
|
|
128
|
+
FROM ( SELECT s_3.measured_at,
|
|
129
|
+
s_3.prev_measured_at,
|
|
130
|
+
s_3.next_measured_at,
|
|
131
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
132
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
133
|
+
CASE
|
|
134
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
135
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
136
|
+
ELSE false
|
|
137
|
+
END AS is_start,
|
|
138
|
+
CASE
|
|
139
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
140
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
141
|
+
ELSE false
|
|
142
|
+
END AS is_end
|
|
143
|
+
FROM ( SELECT s_4.measured_at,
|
|
144
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
145
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
146
|
+
FROM ( SELECT DISTINCT waze_jams.measured_at
|
|
147
|
+
FROM waze_jams WHERE waze_jams.measured_at::double precision >= (( SELECT config.min_measured_at
|
|
148
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
149
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
150
|
+
WHERE s.is_start IS TRUE;
|
|
151
|
+
|
|
152
|
+
CREATE UNIQUE INDEX waze_jams_snapshots_latest_start_measured_at_idx ON waze_jams_snapshots_latest(start_measured_at int8_ops);
|
|
153
|
+
|
|
154
|
+
DROP MATERIALIZED VIEW waze_reconstructions_snapshots_latest;
|
|
155
|
+
|
|
156
|
+
CREATE MATERIALIZED VIEW waze_reconstructions_snapshots_latest AS WITH config AS (
|
|
157
|
+
SELECT 120000 AS max_period,
|
|
158
|
+
date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000::double precision AS min_measured_at
|
|
159
|
+
)
|
|
160
|
+
SELECT s.start_measured_at,
|
|
161
|
+
s.end_measured_at,
|
|
162
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
163
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
164
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
165
|
+
CASE
|
|
166
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
167
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
168
|
+
END + (( SELECT config.max_period
|
|
169
|
+
FROM config)) AS end_measured_at,
|
|
170
|
+
s_1.is_start,
|
|
171
|
+
s_1.is_end
|
|
172
|
+
FROM ( SELECT s_2.measured_at,
|
|
173
|
+
s_2.prev_measured_at,
|
|
174
|
+
s_2.next_measured_at,
|
|
175
|
+
s_2."?column?",
|
|
176
|
+
s_2."?column?_1" AS "?column?",
|
|
177
|
+
s_2.is_start,
|
|
178
|
+
s_2.is_end
|
|
179
|
+
FROM ( SELECT s_3.measured_at,
|
|
180
|
+
s_3.prev_measured_at,
|
|
181
|
+
s_3.next_measured_at,
|
|
182
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
183
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
184
|
+
CASE
|
|
185
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
186
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
187
|
+
ELSE false
|
|
188
|
+
END AS is_start,
|
|
189
|
+
CASE
|
|
190
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
191
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
192
|
+
ELSE false
|
|
193
|
+
END AS is_end
|
|
194
|
+
FROM ( SELECT s_4.measured_at,
|
|
195
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
196
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
197
|
+
FROM ( SELECT DISTINCT waze_reconstructions.measured_at
|
|
198
|
+
FROM waze_reconstructions WHERE waze_reconstructions.measured_at::double precision >= (( SELECT config.min_measured_at
|
|
199
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
200
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
201
|
+
WHERE s.is_start IS TRUE;
|
|
202
|
+
|
|
203
|
+
CREATE UNIQUE INDEX waze_reconstructions_snapshots_latest_start_measured_at_idx ON waze_reconstructions_snapshots_latest(start_measured_at int8_ops);
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
DROP MATERIALIZED VIEW ndic_events_full_snapshots_latest;
|
|
2
|
+
|
|
3
|
+
CREATE MATERIALIZED VIEW ndic_events_full_snapshots_latest AS WITH config AS (
|
|
4
|
+
SELECT 1800000 AS max_period,
|
|
5
|
+
(date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000)::int8 AS min_measured_at
|
|
6
|
+
)
|
|
7
|
+
SELECT s.start_measured_at,
|
|
8
|
+
s.end_measured_at,
|
|
9
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
10
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
11
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
12
|
+
CASE
|
|
13
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
14
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
15
|
+
END + (( SELECT config.max_period
|
|
16
|
+
FROM config)) AS end_measured_at,
|
|
17
|
+
s_1.is_start,
|
|
18
|
+
s_1.is_end
|
|
19
|
+
FROM ( SELECT s_2.measured_at,
|
|
20
|
+
s_2.prev_measured_at,
|
|
21
|
+
s_2.next_measured_at,
|
|
22
|
+
s_2."?column?",
|
|
23
|
+
s_2."?column?_1" AS "?column?",
|
|
24
|
+
s_2.is_start,
|
|
25
|
+
s_2.is_end
|
|
26
|
+
FROM ( SELECT s_3.measured_at,
|
|
27
|
+
s_3.prev_measured_at,
|
|
28
|
+
s_3.next_measured_at,
|
|
29
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
30
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
31
|
+
CASE
|
|
32
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
33
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
34
|
+
ELSE false
|
|
35
|
+
END AS is_start,
|
|
36
|
+
CASE
|
|
37
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
38
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
39
|
+
ELSE false
|
|
40
|
+
END AS is_end
|
|
41
|
+
FROM ( SELECT s_4.measured_at,
|
|
42
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
43
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
44
|
+
FROM ( SELECT DISTINCT ndic_events_full.measured_at
|
|
45
|
+
FROM ndic_events_full WHERE ndic_events_full.measured_at >= (( SELECT config.min_measured_at
|
|
46
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
47
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
48
|
+
WHERE s.is_start IS TRUE;
|
|
49
|
+
|
|
50
|
+
CREATE UNIQUE INDEX ndic_events_full_snapshots_latest_start_measured_at_idx ON ndic_events_full_snapshots_latest(start_measured_at int8_ops);
|
|
51
|
+
|
|
52
|
+
DROP MATERIALIZED VIEW tsk_std_last_30min_snapshots_latest;
|
|
53
|
+
|
|
54
|
+
CREATE MATERIALIZED VIEW tsk_std_last_30min_snapshots_latest AS WITH config AS (
|
|
55
|
+
SELECT 300000 AS max_period,
|
|
56
|
+
(date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000)::int8 AS min_measured_at
|
|
57
|
+
)
|
|
58
|
+
SELECT s.start_measured_at,
|
|
59
|
+
s.end_measured_at,
|
|
60
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
61
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
62
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
63
|
+
CASE
|
|
64
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
65
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
66
|
+
END + (( SELECT config.max_period
|
|
67
|
+
FROM config)) AS end_measured_at,
|
|
68
|
+
s_1.is_start,
|
|
69
|
+
s_1.is_end
|
|
70
|
+
FROM ( SELECT s_2.measured_at,
|
|
71
|
+
s_2.prev_measured_at,
|
|
72
|
+
s_2.next_measured_at,
|
|
73
|
+
s_2."?column?",
|
|
74
|
+
s_2."?column?_1" AS "?column?",
|
|
75
|
+
s_2.is_start,
|
|
76
|
+
s_2.is_end
|
|
77
|
+
FROM ( SELECT s_3.measured_at,
|
|
78
|
+
s_3.prev_measured_at,
|
|
79
|
+
s_3.next_measured_at,
|
|
80
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
81
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
82
|
+
CASE
|
|
83
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
84
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
85
|
+
ELSE false
|
|
86
|
+
END AS is_start,
|
|
87
|
+
CASE
|
|
88
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
89
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
90
|
+
ELSE false
|
|
91
|
+
END AS is_end
|
|
92
|
+
FROM ( SELECT s_4.measured_at,
|
|
93
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
94
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
95
|
+
FROM ( SELECT DISTINCT tsk_std_last_30min.measured_at
|
|
96
|
+
FROM tsk_std_last_30min WHERE tsk_std_last_30min.measured_at >= (( SELECT config.min_measured_at
|
|
97
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
98
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
99
|
+
WHERE s.is_start IS TRUE;
|
|
100
|
+
|
|
101
|
+
CREATE UNIQUE INDEX tsk_std_last_30min_snapshots_latest_start_measured_at_idx ON tsk_std_last_30min_snapshots_latest(start_measured_at int8_ops);
|
|
102
|
+
|
|
103
|
+
DROP MATERIALIZED VIEW waze_jams_snapshots_latest;
|
|
104
|
+
|
|
105
|
+
CREATE MATERIALIZED VIEW waze_jams_snapshots_latest AS WITH config AS (
|
|
106
|
+
SELECT 120000 AS max_period,
|
|
107
|
+
(date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000)::int8 AS min_measured_at
|
|
108
|
+
)
|
|
109
|
+
SELECT s.start_measured_at,
|
|
110
|
+
s.end_measured_at,
|
|
111
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
112
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
113
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
114
|
+
CASE
|
|
115
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
116
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
117
|
+
END + (( SELECT config.max_period
|
|
118
|
+
FROM config)) AS end_measured_at,
|
|
119
|
+
s_1.is_start,
|
|
120
|
+
s_1.is_end
|
|
121
|
+
FROM ( SELECT s_2.measured_at,
|
|
122
|
+
s_2.prev_measured_at,
|
|
123
|
+
s_2.next_measured_at,
|
|
124
|
+
s_2."?column?",
|
|
125
|
+
s_2."?column?_1" AS "?column?",
|
|
126
|
+
s_2.is_start,
|
|
127
|
+
s_2.is_end
|
|
128
|
+
FROM ( SELECT s_3.measured_at,
|
|
129
|
+
s_3.prev_measured_at,
|
|
130
|
+
s_3.next_measured_at,
|
|
131
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
132
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
133
|
+
CASE
|
|
134
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
135
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
136
|
+
ELSE false
|
|
137
|
+
END AS is_start,
|
|
138
|
+
CASE
|
|
139
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
140
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
141
|
+
ELSE false
|
|
142
|
+
END AS is_end
|
|
143
|
+
FROM ( SELECT s_4.measured_at,
|
|
144
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
145
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
146
|
+
FROM ( SELECT DISTINCT waze_jams.measured_at
|
|
147
|
+
FROM waze_jams WHERE waze_jams.measured_at >= (( SELECT config.min_measured_at
|
|
148
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
149
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
150
|
+
WHERE s.is_start IS TRUE;
|
|
151
|
+
|
|
152
|
+
CREATE UNIQUE INDEX waze_jams_snapshots_latest_start_measured_at_idx ON waze_jams_snapshots_latest(start_measured_at int8_ops);
|
|
153
|
+
|
|
154
|
+
DROP MATERIALIZED VIEW waze_reconstructions_snapshots_latest;
|
|
155
|
+
|
|
156
|
+
CREATE MATERIALIZED VIEW waze_reconstructions_snapshots_latest AS WITH config AS (
|
|
157
|
+
SELECT 120000 AS max_period,
|
|
158
|
+
(date_part('epoch'::text, date_trunc('day',now(), 'Europe/Prague')) * 1000)::int8 AS min_measured_at
|
|
159
|
+
)
|
|
160
|
+
SELECT s.start_measured_at,
|
|
161
|
+
s.end_measured_at,
|
|
162
|
+
to_timestamp((s.start_measured_at / 1000)::double precision) AS start_ts,
|
|
163
|
+
to_timestamp((s.end_measured_at / 1000)::double precision) AS end_ts
|
|
164
|
+
FROM ( SELECT s_1.measured_at AS start_measured_at,
|
|
165
|
+
CASE
|
|
166
|
+
WHEN s_1.is_end IS TRUE THEN s_1.measured_at
|
|
167
|
+
ELSE lead(s_1.measured_at) OVER (ORDER BY s_1.measured_at)
|
|
168
|
+
END + (( SELECT config.max_period
|
|
169
|
+
FROM config)) AS end_measured_at,
|
|
170
|
+
s_1.is_start,
|
|
171
|
+
s_1.is_end
|
|
172
|
+
FROM ( SELECT s_2.measured_at,
|
|
173
|
+
s_2.prev_measured_at,
|
|
174
|
+
s_2.next_measured_at,
|
|
175
|
+
s_2."?column?",
|
|
176
|
+
s_2."?column?_1" AS "?column?",
|
|
177
|
+
s_2.is_start,
|
|
178
|
+
s_2.is_end
|
|
179
|
+
FROM ( SELECT s_3.measured_at,
|
|
180
|
+
s_3.prev_measured_at,
|
|
181
|
+
s_3.next_measured_at,
|
|
182
|
+
s_3.measured_at - s_3.prev_measured_at,
|
|
183
|
+
s_3.measured_at - s_3.next_measured_at,
|
|
184
|
+
CASE
|
|
185
|
+
WHEN s_3.prev_measured_at < (s_3.measured_at - (( SELECT config.max_period
|
|
186
|
+
FROM config))) OR s_3.prev_measured_at IS NULL THEN true
|
|
187
|
+
ELSE false
|
|
188
|
+
END AS is_start,
|
|
189
|
+
CASE
|
|
190
|
+
WHEN s_3.next_measured_at > (s_3.measured_at + (( SELECT config.max_period
|
|
191
|
+
FROM config))) OR s_3.next_measured_at IS NULL THEN true
|
|
192
|
+
ELSE false
|
|
193
|
+
END AS is_end
|
|
194
|
+
FROM ( SELECT s_4.measured_at,
|
|
195
|
+
lag(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS prev_measured_at,
|
|
196
|
+
lead(s_4.measured_at) OVER (ORDER BY s_4.measured_at) AS next_measured_at
|
|
197
|
+
FROM ( SELECT DISTINCT waze_reconstructions.measured_at
|
|
198
|
+
FROM waze_reconstructions WHERE waze_reconstructions.measured_at >= (( SELECT config.min_measured_at
|
|
199
|
+
FROM config))) s_4) s_3) s_2(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)
|
|
200
|
+
WHERE s_2.is_start IS TRUE OR s_2.is_end IS TRUE) s_1(measured_at, prev_measured_at, next_measured_at, "?column?", "?column?_1", is_start, is_end)) s
|
|
201
|
+
WHERE s.is_start IS TRUE;
|
|
202
|
+
|
|
203
|
+
CREATE UNIQUE INDEX waze_reconstructions_snapshots_latest_start_measured_at_idx ON waze_reconstructions_snapshots_latest(start_measured_at int8_ops);
|
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
import { AggregationTaskType } from "../schema-definitions/helpers/AggregationTaskType";
|
|
2
2
|
import { DateTime, DurationLike } from "@golemio/core/dist/shared/luxon";
|
|
3
|
-
import { Message } from "amqplib";
|
|
4
3
|
import AggregationTask from "./business/AggregationTask";
|
|
5
4
|
export default class WorkerHelper {
|
|
6
|
-
static parseMessage(message: Message): AggregationTask;
|
|
7
5
|
static generateIntervals: (from: DateTime, to: DateTime, maxDuration: DurationLike) => Array<{
|
|
8
6
|
start: Date;
|
|
9
7
|
end: Date;
|
|
10
8
|
}>;
|
|
11
9
|
static generateSubTasksDefinitions: (from: DateTime, to: DateTime, maxDuration: DurationLike, taskType: AggregationTaskType, updateOnDuplicate: boolean, refreshView: boolean) => AggregationTask[];
|
|
12
|
-
private static parseContent;
|
|
13
10
|
}
|
|
@@ -3,34 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const AggregationTaskType_1 = require("../schema-definitions/helpers/AggregationTaskType");
|
|
7
|
-
const helpers_1 = require("@golemio/core/dist/integration-engine/helpers");
|
|
8
|
-
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
9
|
-
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
|
|
10
6
|
const AggregationTask_1 = __importDefault(require("./business/AggregationTask"));
|
|
11
7
|
class WorkerHelper {
|
|
12
|
-
static parseMessage(message) {
|
|
13
|
-
const task = WorkerHelper.parseContent(message.content.toString());
|
|
14
|
-
if (!task)
|
|
15
|
-
return {};
|
|
16
|
-
const type = AggregationTaskType_1.AggregationTaskType[task.type?.toUpperCase()];
|
|
17
|
-
if (!type)
|
|
18
|
-
throw new golemio_errors_1.GeneralError(`Unknown aggregation task type ${JSON.stringify(task)}`, this.constructor.name);
|
|
19
|
-
if (!task.from || !task.to)
|
|
20
|
-
throw new golemio_errors_1.GeneralError(`Unsupported aggregation task ${JSON.stringify(task)}`, this.constructor.name);
|
|
21
|
-
const result = new AggregationTask_1.default(luxon_1.DateTime.fromISO(task.from).toJSDate(), luxon_1.DateTime.fromISO(task.to).toJSDate(), type, task.updateOnDuplicate ? task.updateOnDuplicate.toString().toLowerCase() == "true" : undefined, task.refreshView ? task.refreshView.toString().toLowerCase() == "true" : undefined);
|
|
22
|
-
if (result.from > result.to)
|
|
23
|
-
throw new golemio_errors_1.GeneralError(`Unsupported task date range ${JSON.stringify(task)}`, this.constructor.name);
|
|
24
|
-
return result;
|
|
25
|
-
}
|
|
26
|
-
static parseContent(messageContent) {
|
|
27
|
-
try {
|
|
28
|
-
return messageContent.length > 0 ? JSON.parse(messageContent) : undefined;
|
|
29
|
-
}
|
|
30
|
-
catch (err) {
|
|
31
|
-
throw new golemio_errors_1.GeneralError(`Unable to parse json from message: ${messageContent}`, this.constructor.name, err);
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
8
|
}
|
|
35
9
|
WorkerHelper.generateIntervals = (from, to, maxDuration) => {
|
|
36
10
|
const breakpoints = [from];
|
|
@@ -54,7 +28,6 @@ WorkerHelper.generateSubTasksDefinitions = (from, to, maxDuration, taskType, upd
|
|
|
54
28
|
const result = [];
|
|
55
29
|
const intervals = WorkerHelper.generateIntervals(from, to, maxDuration);
|
|
56
30
|
for (const item of intervals) {
|
|
57
|
-
helpers_1.log.silly(`generating task from:${item.start.toISOString()}, to:${item.end.toISOString()}`);
|
|
58
31
|
result.push(new AggregationTask_1.default(item.start, item.end, taskType, updateOnDuplicate, refreshView));
|
|
59
32
|
}
|
|
60
33
|
return result;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WorkerHelper.js","sourceRoot":"","sources":["../../src/integration-engine/WorkerHelper.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"WorkerHelper.js","sourceRoot":"","sources":["../../src/integration-engine/WorkerHelper.ts"],"names":[],"mappings":";;;;;AAEA,iFAAyD;AAEzD,MAAqB,YAAY;;AACf,8BAAiB,GAAG,CAC9B,IAAc,EACd,EAAY,EACZ,WAAyB,EACQ,EAAE;IACnC,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAErC,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QACvD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QACrD,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,eAAe,GAAG,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,eAAe,CAAC;AAC3B,CAAC,CAAC;AAEY,wCAA2B,GAAG,CACxC,IAAc,EACd,EAAY,EACZ,WAAyB,EACzB,QAA6B,EAC7B,iBAA0B,EAC1B,WAAoB,EACH,EAAE;IACnB,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,YAAY,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,yBAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC,CAAC;IACrG,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;kBA3Ce,YAAY"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const RUSH_HOUR_AGGREGATION_WORKER_NAME = "rush-hour-aggregation";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/integration-engine/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,iCAAiC,GAAG,uBAAuB,CAAC"}
|
|
@@ -1,8 +1,15 @@
|
|
|
1
|
+
import { IoRedisConnector } from "@golemio/core/dist/helpers/data-access/redis/IoRedisConnector";
|
|
2
|
+
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
1
3
|
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine";
|
|
4
|
+
import { ISequelizeSettings } from "@golemio/core/dist/integration-engine/models/IModel";
|
|
5
|
+
import { IValidator } from "@golemio/core/dist/shared/golemio-validator";
|
|
2
6
|
export default abstract class EventsRepository<T> extends PostgresModel implements IModel {
|
|
7
|
+
private logger;
|
|
8
|
+
private redisConnector;
|
|
3
9
|
private readonly lockTimeout;
|
|
4
10
|
protected abstract materializedViewNameLatest: string;
|
|
5
11
|
protected abstract materializedViewNameHistoric: string;
|
|
12
|
+
constructor(name: string, settings: ISequelizeSettings, validator: IValidator, logger: ILogger, redisConnector: IoRedisConnector);
|
|
6
13
|
abstract saveData(data: T[], updateOnDuplicate: boolean): Promise<void>;
|
|
7
14
|
refreshLatestMaterializedView: () => Promise<void>;
|
|
8
15
|
refreshHistoricMaterializedView: () => Promise<void>;
|
|
@@ -5,11 +5,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
const PrahaDopravniInfo_1 = __importDefault(require("../../schema-definitions/sequelize-models/PrahaDopravniInfo"));
|
|
7
7
|
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
8
|
-
const helpers_1 = require("@golemio/core/dist/integration-engine/helpers");
|
|
9
8
|
const redis_semaphore_1 = require("@golemio/core/dist/shared/redis-semaphore");
|
|
10
9
|
class EventsRepository extends integration_engine_1.PostgresModel {
|
|
11
|
-
constructor() {
|
|
12
|
-
super(
|
|
10
|
+
constructor(name, settings, validator, logger, redisConnector) {
|
|
11
|
+
super(name, settings, validator);
|
|
12
|
+
this.logger = logger;
|
|
13
|
+
this.redisConnector = redisConnector;
|
|
13
14
|
this.lockTimeout = 4 * 60 * 60 * 1000; // 4 hours
|
|
14
15
|
this.refreshLatestMaterializedView = async () => {
|
|
15
16
|
await this.refreshMaterializedView(this.materializedViewNameLatest);
|
|
@@ -18,16 +19,18 @@ class EventsRepository extends integration_engine_1.PostgresModel {
|
|
|
18
19
|
await this.refreshMaterializedView(this.materializedViewNameHistoric);
|
|
19
20
|
};
|
|
20
21
|
this.refreshMaterializedView = async (viewName) => {
|
|
21
|
-
|
|
22
|
-
const semaphore = this.createSemaphore(
|
|
22
|
+
this.logger.debug(`Refreshing materialized view: ${viewName}`);
|
|
23
|
+
const semaphore = this.createSemaphore(viewName);
|
|
23
24
|
const lockAcquired = await semaphore.tryAcquire();
|
|
24
25
|
if (!lockAcquired) {
|
|
25
|
-
|
|
26
|
+
this.logger.info(`Rush Hour Aggregation - unable to obtain mutex lock for: ${viewName}`);
|
|
26
27
|
return;
|
|
27
28
|
}
|
|
28
29
|
try {
|
|
29
30
|
const sql = `REFRESH MATERIALIZED VIEW CONCURRENTLY ${PrahaDopravniInfo_1.default.SCHEMA}.${viewName}`;
|
|
31
|
+
const startedAt = Date.now();
|
|
30
32
|
await this.query(sql);
|
|
33
|
+
this.logger.info(`Rush Hour Aggregation - refreshed materialized view ${viewName} in ${Date.now() - startedAt} ms.`);
|
|
31
34
|
}
|
|
32
35
|
finally {
|
|
33
36
|
await semaphore.release();
|
|
@@ -35,13 +38,13 @@ class EventsRepository extends integration_engine_1.PostgresModel {
|
|
|
35
38
|
};
|
|
36
39
|
}
|
|
37
40
|
createSemaphore(keyPhrase) {
|
|
38
|
-
const redisClient =
|
|
41
|
+
const redisClient = this.redisConnector.getConnection();
|
|
39
42
|
return new redis_semaphore_1.Mutex(redisClient, `RushHourAggregation-${keyPhrase}`, {
|
|
40
43
|
acquireAttemptsLimit: 1,
|
|
41
44
|
lockTimeout: this.lockTimeout,
|
|
42
45
|
refreshInterval: this.lockTimeout * 0.8,
|
|
43
46
|
onLockLost: (err) => {
|
|
44
|
-
|
|
47
|
+
this.logger.warn(`RushHourAggregation - ${err.message}`);
|
|
45
48
|
},
|
|
46
49
|
});
|
|
47
50
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractEventsRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/AbstractEventsRepository.ts"],"names":[],"mappings":";;;;;AAAA,oHAAwE;
|
|
1
|
+
{"version":3,"file":"AbstractEventsRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/AbstractEventsRepository.ts"],"names":[],"mappings":";;;;;AAAA,oHAAwE;AAGxE,8EAA8E;AAG9E,+EAAkE;AAElE,MAA8B,gBAAoB,SAAQ,kCAAa;IAMnE,YACI,IAAY,EACZ,QAA4B,EAC5B,SAAqB,EACb,MAAe,EACf,cAAgC;QAExC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAHzB,WAAM,GAAN,MAAM,CAAS;QACf,mBAAc,GAAd,cAAc,CAAkB;QAV3B,gBAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;QAiBtD,kCAA6B,GAAG,KAAK,IAAmB,EAAE;YAC7D,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxE,CAAC,CAAC;QAEK,oCAA+B,GAAG,KAAK,IAAmB,EAAE;YAC/D,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1E,CAAC,CAAC;QAEM,4BAAuB,GAAG,KAAK,EAAE,QAAgB,EAAiB,EAAE;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAC;YAE/D,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC;YAClD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4DAA4D,QAAQ,EAAE,CAAC,CAAC;gBACzF,OAAO;YACX,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,0CAA0C,2BAAiB,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7F,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE7B,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,QAAQ,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,MAAM,CAAC,CAAC;YACzH,CAAC;oBAAS,CAAC;gBACP,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC;QACL,CAAC,CAAC;IA/BF,CAAC;IAiCO,eAAe,CAAC,SAAiB;QACrC,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;QACxD,OAAO,IAAI,uBAAK,CAAC,WAAW,EAAE,uBAAuB,SAAS,EAAE,EAAE;YAC9D,oBAAoB,EAAE,CAAC;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,eAAe,EAAE,IAAI,CAAC,WAAW,GAAG,GAAG;YACvC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7D,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ;AA1DD,mCA0DC"}
|