@golemio/pid 2.13.4-dev.1297347910 → 2.13.4-dev.1300348576
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/postgresql/20240519214722-data-retention-valid-to-check.js +53 -0
- package/db/migrations/postgresql/sqls/20240519214722-data-retention-valid-to-check-down.sql +76 -0
- package/db/migrations/postgresql/sqls/20240519214722-data-retention-valid-to-check-up.sql +76 -0
- package/docs/implementation_documentation.md +7 -42
- package/package.json +2 -2
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dbm;
|
|
4
|
+
var type;
|
|
5
|
+
var seed;
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var Promise;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* We receive the dbmigrate dependency from dbmigrate initially.
|
|
12
|
+
* This enables us to not have to rely on NODE_PATH.
|
|
13
|
+
*/
|
|
14
|
+
exports.setup = function(options, seedLink) {
|
|
15
|
+
dbm = options.dbmigrate;
|
|
16
|
+
type = dbm.dataType;
|
|
17
|
+
seed = seedLink;
|
|
18
|
+
Promise = options.Promise;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
exports.up = function(db) {
|
|
22
|
+
var filePath = path.join(__dirname, 'sqls', '20240519214722-data-retention-valid-to-check-up.sql');
|
|
23
|
+
return new Promise( function( resolve, reject ) {
|
|
24
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
25
|
+
if (err) return reject(err);
|
|
26
|
+
console.log('received data: ' + data);
|
|
27
|
+
|
|
28
|
+
resolve(data);
|
|
29
|
+
});
|
|
30
|
+
})
|
|
31
|
+
.then(function(data) {
|
|
32
|
+
return db.runSql(data);
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
exports.down = function(db) {
|
|
37
|
+
var filePath = path.join(__dirname, 'sqls', '20240519214722-data-retention-valid-to-check-down.sql');
|
|
38
|
+
return new Promise( function( resolve, reject ) {
|
|
39
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
40
|
+
if (err) return reject(err);
|
|
41
|
+
console.log('received data: ' + data);
|
|
42
|
+
|
|
43
|
+
resolve(data);
|
|
44
|
+
});
|
|
45
|
+
})
|
|
46
|
+
.then(function(data) {
|
|
47
|
+
return db.runSql(data);
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
exports._meta = {
|
|
52
|
+
"version": 1
|
|
53
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
CREATE OR REPLACE PROCEDURE vehiclepositions_data_retention(inout numberOfRows int, in dataRetentionMinutes int)
|
|
2
|
+
LANGUAGE plpgsql
|
|
3
|
+
SET search_path FROM CURRENT
|
|
4
|
+
AS $procedure$
|
|
5
|
+
declare
|
|
6
|
+
idsForDelete varchar(255)[];
|
|
7
|
+
begin
|
|
8
|
+
select array_agg(t.id) from vehiclepositions_trips t
|
|
9
|
+
left join vehiclepositions_positions p on
|
|
10
|
+
p.id = t.last_position_id
|
|
11
|
+
where
|
|
12
|
+
p.valid_to < (NOW() - (dataRetentionMinutes || ' minutes')::interval)
|
|
13
|
+
or (gtfs_trip_id is null and p.created_at < NOW() - (dataRetentionMinutes || ' minutes')::interval) -- entries with valid to is null
|
|
14
|
+
into idsForDelete;
|
|
15
|
+
|
|
16
|
+
INSERT INTO vehiclepositions_trips_history
|
|
17
|
+
select * from vehiclepositions_trips where id = ANY(idsForDelete)
|
|
18
|
+
on conflict do nothing;
|
|
19
|
+
|
|
20
|
+
INSERT INTO vehiclepositions_positions_history
|
|
21
|
+
select * from vehiclepositions_positions where trips_id = ANY(idsForDelete)
|
|
22
|
+
on conflict do nothing;
|
|
23
|
+
|
|
24
|
+
insert into vehiclepositions_stop_times_history (
|
|
25
|
+
rt_trip_id,
|
|
26
|
+
gtfs_date,
|
|
27
|
+
gtfs_trip_id,
|
|
28
|
+
gtfs_direction_id,
|
|
29
|
+
gtfs_route_short_name,
|
|
30
|
+
gtfs_route_type,
|
|
31
|
+
run_number,
|
|
32
|
+
vehicle_registration_number,
|
|
33
|
+
gtfs_stop_id,
|
|
34
|
+
gtfs_stop_sequence,
|
|
35
|
+
current_stop_arrival,
|
|
36
|
+
current_stop_departure,
|
|
37
|
+
current_stop_arr_delay,
|
|
38
|
+
current_stop_dep_delay,
|
|
39
|
+
created_at,
|
|
40
|
+
updated_at,
|
|
41
|
+
origin_route_name
|
|
42
|
+
)
|
|
43
|
+
select
|
|
44
|
+
rt_trip.id,
|
|
45
|
+
rt_trip.gtfs_date,
|
|
46
|
+
rt_trip.gtfs_trip_id,
|
|
47
|
+
rt_trip.gtfs_direction_id,
|
|
48
|
+
rt_trip.gtfs_route_short_name,
|
|
49
|
+
rt_trip.gtfs_route_type,
|
|
50
|
+
rt_trip.run_number,
|
|
51
|
+
rt_trip.vehicle_registration_number,
|
|
52
|
+
stop_time.stop_id,
|
|
53
|
+
stop_time.stop_sequence,
|
|
54
|
+
stop_time.stop_arrival,
|
|
55
|
+
stop_time.stop_departure,
|
|
56
|
+
stop_time.stop_arr_delay,
|
|
57
|
+
stop_time.stop_dep_delay,
|
|
58
|
+
now(),
|
|
59
|
+
now(),
|
|
60
|
+
rt_trip.origin_route_name
|
|
61
|
+
from
|
|
62
|
+
vehiclepositions_trips rt_trip
|
|
63
|
+
join
|
|
64
|
+
v_vehiclepositions_past_stop_times stop_time on stop_time.rt_trip_id = rt_trip.id
|
|
65
|
+
where
|
|
66
|
+
rt_trip.id = ANY(idsForDelete)
|
|
67
|
+
on conflict do nothing;
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
delete from vehiclepositions_positions where trips_id = ANY(idsForDelete);
|
|
71
|
+
delete from vehiclepositions_trips where id = ANY(idsForDelete);
|
|
72
|
+
delete from vehiclepositions_cis_stops where rt_trip_id = ANY(idsForDelete);
|
|
73
|
+
|
|
74
|
+
select array_length(idsForDelete,1) into numberOfRows;
|
|
75
|
+
end;
|
|
76
|
+
$procedure$;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
CREATE OR REPLACE PROCEDURE vehiclepositions_data_retention(inout numberOfRows int, in dataRetentionMinutes int)
|
|
2
|
+
LANGUAGE plpgsql
|
|
3
|
+
SET search_path FROM CURRENT
|
|
4
|
+
AS $procedure$
|
|
5
|
+
declare
|
|
6
|
+
idsForDelete varchar(255)[];
|
|
7
|
+
begin
|
|
8
|
+
select array_agg(t.id) from vehiclepositions_trips t
|
|
9
|
+
left join vehiclepositions_positions p on
|
|
10
|
+
p.id = t.last_position_id
|
|
11
|
+
where
|
|
12
|
+
p.valid_to < (NOW() - (dataRetentionMinutes || ' minutes')::interval)
|
|
13
|
+
or (t.gtfs_trip_id is null and p.valid_to is null and p.updated_at < NOW() - (dataRetentionMinutes || ' minutes')::interval) -- entries with valid to is null
|
|
14
|
+
into idsForDelete;
|
|
15
|
+
|
|
16
|
+
INSERT INTO vehiclepositions_trips_history
|
|
17
|
+
select * from vehiclepositions_trips where id = ANY(idsForDelete)
|
|
18
|
+
on conflict do nothing;
|
|
19
|
+
|
|
20
|
+
INSERT INTO vehiclepositions_positions_history
|
|
21
|
+
select * from vehiclepositions_positions where trips_id = ANY(idsForDelete)
|
|
22
|
+
on conflict do nothing;
|
|
23
|
+
|
|
24
|
+
insert into vehiclepositions_stop_times_history (
|
|
25
|
+
rt_trip_id,
|
|
26
|
+
gtfs_date,
|
|
27
|
+
gtfs_trip_id,
|
|
28
|
+
gtfs_direction_id,
|
|
29
|
+
gtfs_route_short_name,
|
|
30
|
+
gtfs_route_type,
|
|
31
|
+
run_number,
|
|
32
|
+
vehicle_registration_number,
|
|
33
|
+
gtfs_stop_id,
|
|
34
|
+
gtfs_stop_sequence,
|
|
35
|
+
current_stop_arrival,
|
|
36
|
+
current_stop_departure,
|
|
37
|
+
current_stop_arr_delay,
|
|
38
|
+
current_stop_dep_delay,
|
|
39
|
+
created_at,
|
|
40
|
+
updated_at,
|
|
41
|
+
origin_route_name
|
|
42
|
+
)
|
|
43
|
+
select
|
|
44
|
+
rt_trip.id,
|
|
45
|
+
rt_trip.gtfs_date,
|
|
46
|
+
rt_trip.gtfs_trip_id,
|
|
47
|
+
rt_trip.gtfs_direction_id,
|
|
48
|
+
rt_trip.gtfs_route_short_name,
|
|
49
|
+
rt_trip.gtfs_route_type,
|
|
50
|
+
rt_trip.run_number,
|
|
51
|
+
rt_trip.vehicle_registration_number,
|
|
52
|
+
stop_time.stop_id,
|
|
53
|
+
stop_time.stop_sequence,
|
|
54
|
+
stop_time.stop_arrival,
|
|
55
|
+
stop_time.stop_departure,
|
|
56
|
+
stop_time.stop_arr_delay,
|
|
57
|
+
stop_time.stop_dep_delay,
|
|
58
|
+
now(),
|
|
59
|
+
now(),
|
|
60
|
+
rt_trip.origin_route_name
|
|
61
|
+
from
|
|
62
|
+
vehiclepositions_trips rt_trip
|
|
63
|
+
join
|
|
64
|
+
v_vehiclepositions_past_stop_times stop_time on stop_time.rt_trip_id = rt_trip.id
|
|
65
|
+
where
|
|
66
|
+
rt_trip.id = ANY(idsForDelete)
|
|
67
|
+
on conflict do nothing;
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
delete from vehiclepositions_positions where trips_id = ANY(idsForDelete);
|
|
71
|
+
delete from vehiclepositions_trips where id = ANY(idsForDelete);
|
|
72
|
+
delete from vehiclepositions_cis_stops where rt_trip_id = ANY(idsForDelete);
|
|
73
|
+
|
|
74
|
+
select array_length(idsForDelete,1) into numberOfRows;
|
|
75
|
+
end;
|
|
76
|
+
$procedure$;
|
|
@@ -46,7 +46,7 @@ Modul slouží k ukládání a poskytování informací o jízdních řádech a
|
|
|
46
46
|
- příklad vstupnich dat https://gitlab.com/operator-ict/golemio/code/modules/pid/-/snippets/2462385
|
|
47
47
|
- frekvence stahování
|
|
48
48
|
- cron definice:
|
|
49
|
-
- cron.vehicle-positions.ropidgtfs
|
|
49
|
+
- cron.vehicle-positions.ropidgtfs.checkForNewData
|
|
50
50
|
- rabin `0 35 5,12 * * *`
|
|
51
51
|
- prod `0 20 5 * * *`, `0 50 12 * * *`
|
|
52
52
|
- názvy rabbitmq front
|
|
@@ -140,12 +140,12 @@ _:warning: Původním záměrem bylo využití OIS číselníku během zpracová
|
|
|
140
140
|
- příklad vstupnich dat: [ropiddeparturespresets-data.json](https://gitlab.com/operator-ict/golemio/code/modules/pid/-/blob/development/test/integration-engine/ropid-gtfs/data/ropiddeparturespresets-data.json)
|
|
141
141
|
- frekvence stahování
|
|
142
142
|
- cron definice:
|
|
143
|
-
- cron.vehicle-positions.
|
|
143
|
+
- cron.vehicle-positions.ropidpresets.checkForNewDeparturesPresets
|
|
144
144
|
- rabin `15 */4 * * * *`
|
|
145
145
|
- prod `15 */4 * * * *`
|
|
146
146
|
- názvy rabbitmq front
|
|
147
|
-
- vehicle-positions.
|
|
148
|
-
- vehicle-positions.
|
|
147
|
+
- vehicle-positions.ropidpresets.checkForNewDeparturesPresets
|
|
148
|
+
- vehicle-positions.ropidpresets.downloadDeparturesPresets
|
|
149
149
|
|
|
150
150
|
#### _VYMI výluky a mimořádnosti_
|
|
151
151
|
|
|
@@ -394,41 +394,6 @@ _:warning: Původním záměrem bylo využití OIS číselníku během zpracová
|
|
|
394
394
|
|
|
395
395
|
Všechny tabulky se nachází ve schématu `pid`
|
|
396
396
|
|
|
397
|
-
### _LegacyRopidGTFSWorker_
|
|
398
|
-
|
|
399
|
-
Worker má na starost aktualizaci presets číselníků
|
|
400
|
-
|
|
401
|
-
#### _checkForNewDeparturesPresets()_
|
|
402
|
-
|
|
403
|
-
- vstupní rabbitmq fronta
|
|
404
|
-
- název: vehicle-positions.ropidgtfs.checkForNewDeparturesPresets
|
|
405
|
-
- TTL: 4 minuty
|
|
406
|
-
- parametry: žádné
|
|
407
|
-
- závislé fronty (do kterých jsou odesílány zprávy z metody workeru)
|
|
408
|
-
- název: vehicle-positions.ropidgtfs.downloadDeparturesPresets
|
|
409
|
-
- datové zdroje
|
|
410
|
-
- ROPID FTP
|
|
411
|
-
- transformace
|
|
412
|
-
- žádné
|
|
413
|
-
- data modely
|
|
414
|
-
- RopidGTFSMetadataModel `ropidgtfs_metadata`
|
|
415
|
-
|
|
416
|
-
#### _downloadDeparturesPresets()_
|
|
417
|
-
|
|
418
|
-
- vstupní rabbitmq fronta
|
|
419
|
-
- název: vehicle-positions.ropidgtfs.downloadDeparturesPresets
|
|
420
|
-
- TTL: 4 minuty
|
|
421
|
-
- parametry: žádné
|
|
422
|
-
- závislé fronty (do kterých jsou odesílány zprávy z metody workeru)
|
|
423
|
-
- žádné
|
|
424
|
-
- datové zdroje
|
|
425
|
-
- ROPID FTP
|
|
426
|
-
- transformace
|
|
427
|
-
- [RopidDeparturesPresetsTransformation](https://gitlab.com/operator-ict/golemio/code/modules/pid/-/blob/development/src/integration-engine/ropid-gtfs/transformations/RopidDeparturesPresetsTransformation.ts)
|
|
428
|
-
- data modely
|
|
429
|
-
- RopidGTFSMetadataModel `ropidgtfs_metadata`,
|
|
430
|
-
- RopidDeparturesPresetsModel `ropid_departures_presets`
|
|
431
|
-
|
|
432
397
|
### TimetableWorker
|
|
433
398
|
|
|
434
399
|
#### task _CheckForNewData_
|
|
@@ -619,11 +584,11 @@ Task se stará o aktualizaci cache pro public odjezdy. Pouští se pravidelně c
|
|
|
619
584
|
#### _task: CheckForNewDeparturesPresetsTask_
|
|
620
585
|
|
|
621
586
|
- vstupní rabbitmq fronta
|
|
622
|
-
- název: vehicle-positions.
|
|
587
|
+
- název: vehicle-positions.ropidpresets.checkForNewDeparturesPresets
|
|
623
588
|
- TTL: 4 minuty
|
|
624
589
|
- parametry: žádné
|
|
625
590
|
- závislé fronty (do kterých jsou odesílány zprávy z metody workeru)
|
|
626
|
-
- název: vehicle-positions.
|
|
591
|
+
- název: vehicle-positions.ropidpresets.downloadDeparturesPresets
|
|
627
592
|
- datové zdroje
|
|
628
593
|
- ROPID FTP
|
|
629
594
|
- transformace
|
|
@@ -634,7 +599,7 @@ Task se stará o aktualizaci cache pro public odjezdy. Pouští se pravidelně c
|
|
|
634
599
|
#### _DownloadDeparturesPresetsTask_
|
|
635
600
|
|
|
636
601
|
- vstupní rabbitmq fronta
|
|
637
|
-
- název: vehicle-positions.
|
|
602
|
+
- název: vehicle-positions.ropidpresets.downloadDeparturesPresets
|
|
638
603
|
- TTL: 4 minuty
|
|
639
604
|
- parametry: žádné
|
|
640
605
|
- závislé fronty (do kterých jsou odesílány zprávy z metody workeru)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/pid",
|
|
3
|
-
"version": "2.13.4-dev.
|
|
3
|
+
"version": "2.13.4-dev.1300348576",
|
|
4
4
|
"description": "Golemio PID Module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"@commitlint/cli": "^11.0.0",
|
|
38
38
|
"@commitlint/config-conventional": "^11.0.0",
|
|
39
39
|
"@golemio/cli": "1.5.0",
|
|
40
|
-
"@golemio/core": "1.10.
|
|
40
|
+
"@golemio/core": "1.10.2",
|
|
41
41
|
"@golemio/db-common": "1.1.4",
|
|
42
42
|
"@golemio/eslint-config": "1.1.2",
|
|
43
43
|
"@types/amqplib": "^0.5.17",
|