@golemio/rush-hour-aggregation 1.1.5 → 1.1.6-dev.984077328
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/postgresql/20230827092408-add-data-retention.js +53 -0
- package/db/migrations/postgresql/sqls/20230827092408-add-data-retention-down.sql +1 -0
- package/db/migrations/postgresql/sqls/20230827092408-add-data-retention-up.sql +15 -0
- package/dist/integration-engine/AggregationWorker.d.ts +1 -0
- package/dist/integration-engine/AggregationWorker.js +6 -3
- package/dist/integration-engine/AggregationWorker.js.map +1 -1
- package/dist/integration-engine/dataAccess/AbstractEventsRepository.d.ts +2 -0
- package/dist/integration-engine/dataAccess/AbstractEventsRepository.js +26 -2
- package/dist/integration-engine/dataAccess/AbstractEventsRepository.js.map +1 -1
- package/dist/integration-engine/dataAccess/DataRetentionRepository.d.ts +9 -0
- package/dist/integration-engine/dataAccess/DataRetentionRepository.js +33 -0
- package/dist/integration-engine/dataAccess/DataRetentionRepository.js.map +1 -0
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.d.ts +2 -0
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.js +1 -0
- package/dist/integration-engine/service/aggregators/WazeReconstructionsAggregator.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dbm;
|
|
4
|
+
var type;
|
|
5
|
+
var seed;
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var Promise;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* We receive the dbmigrate dependency from dbmigrate initially.
|
|
12
|
+
* This enables us to not have to rely on NODE_PATH.
|
|
13
|
+
*/
|
|
14
|
+
exports.setup = function(options, seedLink) {
|
|
15
|
+
dbm = options.dbmigrate;
|
|
16
|
+
type = dbm.dataType;
|
|
17
|
+
seed = seedLink;
|
|
18
|
+
Promise = options.Promise;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
exports.up = function(db) {
|
|
22
|
+
var filePath = path.join(__dirname, 'sqls', '20230827092408-add-data-retention-up.sql');
|
|
23
|
+
return new Promise( function( resolve, reject ) {
|
|
24
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
25
|
+
if (err) return reject(err);
|
|
26
|
+
console.log('received data: ' + data);
|
|
27
|
+
|
|
28
|
+
resolve(data);
|
|
29
|
+
});
|
|
30
|
+
})
|
|
31
|
+
.then(function(data) {
|
|
32
|
+
return db.runSql(data);
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
exports.down = function(db) {
|
|
37
|
+
var filePath = path.join(__dirname, 'sqls', '20230827092408-add-data-retention-down.sql');
|
|
38
|
+
return new Promise( function( resolve, reject ) {
|
|
39
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
40
|
+
if (err) return reject(err);
|
|
41
|
+
console.log('received data: ' + data);
|
|
42
|
+
|
|
43
|
+
resolve(data);
|
|
44
|
+
});
|
|
45
|
+
})
|
|
46
|
+
.then(function(data) {
|
|
47
|
+
return db.runSql(data);
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
exports._meta = {
|
|
52
|
+
"version": 1
|
|
53
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
drop procedure data_retention();
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
CREATE OR REPLACE PROCEDURE data_retention()
|
|
2
|
+
LANGUAGE plpgsql
|
|
3
|
+
SET search_path FROM CURRENT
|
|
4
|
+
AS $procedure$
|
|
5
|
+
declare
|
|
6
|
+
olderThanEpoch int8;
|
|
7
|
+
begin
|
|
8
|
+
select extract(epoch from (current_date - interval '3 months')) * 1000 into olderThanEpoch;
|
|
9
|
+
delete from fcd_events where measured_at < olderThanEpoch;
|
|
10
|
+
delete from ndic_events_full where measured_at < olderThanEpoch;
|
|
11
|
+
delete from tsk_std_last_30min where measured_at < olderThanEpoch;
|
|
12
|
+
delete from waze_jams where measured_at < olderThanEpoch;
|
|
13
|
+
delete from waze_reconstructions where measured_at < olderThanEpoch;
|
|
14
|
+
end
|
|
15
|
+
$procedure$;
|
|
@@ -6,6 +6,7 @@ export declare class AggregationWorker extends BaseWorker {
|
|
|
6
6
|
static refreshHistoricViewsMethodName: string;
|
|
7
7
|
private aggregationFactory;
|
|
8
8
|
private refreshIntervalInMinutes;
|
|
9
|
+
private dataRetentionRepository;
|
|
9
10
|
constructor();
|
|
10
11
|
aggregateData: (message: Message) => Promise<void>;
|
|
11
12
|
refreshHistoricViews: () => Promise<void>;
|
|
@@ -13,13 +13,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.AggregationWorker = void 0;
|
|
16
|
+
const AggregationTaskType_1 = require("../schema-definitions/helpers/AggregationTaskType");
|
|
16
17
|
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
17
18
|
const config_1 = require("@golemio/core/dist/integration-engine/config");
|
|
18
|
-
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
19
19
|
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
|
|
20
|
-
const
|
|
21
|
-
const AggregationFactory_1 = __importDefault(require("./service/AggregationFactory"));
|
|
20
|
+
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
22
21
|
const WorkerHelper_1 = __importDefault(require("./WorkerHelper"));
|
|
22
|
+
const DataRetentionRepository_1 = require("./dataAccess/DataRetentionRepository");
|
|
23
|
+
const AggregationFactory_1 = __importDefault(require("./service/AggregationFactory"));
|
|
23
24
|
class AggregationWorker extends integration_engine_1.BaseWorker {
|
|
24
25
|
constructor() {
|
|
25
26
|
super();
|
|
@@ -41,6 +42,7 @@ class AggregationWorker extends integration_engine_1.BaseWorker {
|
|
|
41
42
|
}
|
|
42
43
|
});
|
|
43
44
|
this.refreshHistoricViews = () => __awaiter(this, void 0, void 0, function* () {
|
|
45
|
+
yield this.dataRetentionRepository.deleteOldData();
|
|
44
46
|
for (const type in AggregationTaskType_1.AggregationTaskType) {
|
|
45
47
|
try {
|
|
46
48
|
const aggregator = this.aggregationFactory.get(type.toUpperCase());
|
|
@@ -74,6 +76,7 @@ class AggregationWorker extends integration_engine_1.BaseWorker {
|
|
|
74
76
|
}
|
|
75
77
|
});
|
|
76
78
|
this.aggregationFactory = new AggregationFactory_1.default();
|
|
79
|
+
this.dataRetentionRepository = new DataRetentionRepository_1.DataRetentionRepository(integration_engine_1.PostgresConnector, integration_engine_1.log);
|
|
77
80
|
}
|
|
78
81
|
isTaskEmpty(task) {
|
|
79
82
|
return !task.from && !task.to && !task.type;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AggregationWorker.js","sourceRoot":"","sources":["../../src/integration-engine/AggregationWorker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"AggregationWorker.js","sourceRoot":"","sources":["../../src/integration-engine/AggregationWorker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2FAAuE;AACvE,8EAA2F;AAC3F,yEAAsE;AACtE,6EAAwE;AACxE,2DAAqE;AAErE,kEAA0C;AAE1C,kFAA+E;AAC/E,sFAA8D;AAE9D,MAAa,iBAAkB,SAAQ,+BAAU;IAQ7C;QACI,KAAK,EAAE,CAAC;QAJJ,6BAAwB,GAAW,EAAE,CAAC;QASvC,kBAAa,GAAG,CAAO,OAAgB,EAAE,EAAE;YAC9C,MAAM,IAAI,GAAG,sBAAY,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAChD,wBAAG,CAAC,KAAK,CAAC,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE;gBACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;iBAAM,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE;gBAClC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;aACrC;iBAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE;gBAC1C,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aACnG;iBAAM;gBACH,MAAM,IAAI,6BAAY,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACrG;QACL,CAAC,CAAA,CAAC;QAEK,yBAAoB,GAAG,GAAS,EAAE;YACrC,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,CAAC;YAEnD,KAAK,MAAM,IAAI,IAAI,yCAAmB,EAAE;gBACpC,IAAI;oBACA,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAyB,CAAC,CAAC;oBAC1F,MAAM,UAAU,CAAC,2BAA2B,EAAE,CAAC;iBAClD;gBAAC,OAAO,GAAG,EAAE;oBACV,wBAAG,CAAC,KAAK,CAAC,iDAAiD,IAAI,kBAAkB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;iBACjG;aACJ;QACL,CAAC,CAAA,CAAC;QAiBM,oBAAe,GAAG,GAAwB,EAAE;YAChD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,gBAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClG,KAAK,MAAM,IAAI,IAAI,yCAAmB,EAAE;gBACpC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;aACtD;QACL,CAAC,CAAA,CAAC;QAEM,gBAAW,GAAG,CAClB,IAAY,EACZ,IAAU,EACV,EAAQ,EACR,iBAA0B,EAC1B,WAAoB,EACP,EAAE;YACf,IAAI;gBACA,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAyB,CAAC,CAAC;gBAC1F,MAAM,UAAU,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;aAC/E;YAAC,OAAO,GAAG,EAAE;gBACV,wBAAG,CAAC,KAAK,CAAC,2BAA2B,IAAI,UAAU,IAAI,QAAQ,EAAE,MAAM,GAAG,CAAC,QAAQ,EAAE,kBAAkB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;aACvH;QACL,CAAC,CAAA,CAAC;QAEM,qBAAgB,GAAG,CAAO,IAAqB,EAAiB,EAAE;YACtE,MAAM,QAAQ,GAAG,sBAAY,CAAC,2BAA2B,CACrD,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAK,CAAC,EAC/B,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAG,CAAC,EAC7B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAK,CAAC,CAAC,cAAc,EAAE,EACxD,IAAI,CAAC,IAAK,EACV,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,WAAW,CACnB,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC5B,MAAM,IAAI,CAAC,qBAAqB,CAC5B,UAAU,GAAG,iBAAiB,CAAC,WAAW,GAAG,GAAG,GAAG,iBAAiB,CAAC,qBAAqB,EAC1F,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAC1B,CAAC;aACL;QACL,CAAC,CAAA,CAAC;QArFE,IAAI,CAAC,kBAAkB,GAAG,IAAI,4BAAkB,EAAE,CAAC;QACnD,IAAI,CAAC,uBAAuB,GAAG,IAAI,iDAAuB,CAAC,sCAAiB,EAAE,wBAAG,CAAC,CAAC;IACvF,CAAC;IA6BO,WAAW,CAAC,IAAqB;QACrC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAChD,CAAC;IAEO,cAAc,CAAC,IAAqB;QACxC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE;YACnC,MAAM,WAAW,GAAG,gBAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;YACvG,MAAM,UAAU,GAAG,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAE9F,OAAO,UAAU,IAAI,UAAU,GAAG,WAAW,CAAC;SACjD;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;;AAtDL,8CAgGC;AA/FiB,6BAAW,GAAG,eAAM,CAAC,oBAAoB,GAAG,GAAG,GAAG,uBAAuB,AAA9D,CAA+D;AAC1E,uCAAqB,GAAG,eAAe,AAAlB,CAAmB;AACxC,gDAA8B,GAAG,sBAAsB,AAAzB,CAA0B"}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine";
|
|
2
2
|
export default abstract class EventsRepository<T> extends PostgresModel implements IModel {
|
|
3
|
+
private readonly lockTimeout;
|
|
3
4
|
protected abstract materializedViewNameLatest: string;
|
|
4
5
|
protected abstract materializedViewNameHistoric: string;
|
|
5
6
|
abstract saveData(data: T[], updateOnDuplicate: boolean): Promise<void>;
|
|
6
7
|
refreshLatestMaterializedView: () => Promise<void>;
|
|
7
8
|
refreshHistoricMaterializedView: () => Promise<void>;
|
|
8
9
|
private refreshMaterializedView;
|
|
10
|
+
private createSemaphore;
|
|
9
11
|
}
|
|
@@ -15,9 +15,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
15
15
|
const PrahaDopravniInfo_1 = __importDefault(require("../../schema-definitions/sequelize-models/PrahaDopravniInfo"));
|
|
16
16
|
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
17
17
|
const helpers_1 = require("@golemio/core/dist/integration-engine/helpers");
|
|
18
|
+
const redis_semaphore_1 = require("@golemio/core/dist/shared/redis-semaphore");
|
|
18
19
|
class EventsRepository extends integration_engine_1.PostgresModel {
|
|
19
20
|
constructor() {
|
|
20
21
|
super(...arguments);
|
|
22
|
+
this.lockTimeout = 4 * 60 * 60 * 1000; // 4 hours
|
|
21
23
|
this.refreshLatestMaterializedView = () => __awaiter(this, void 0, void 0, function* () {
|
|
22
24
|
yield this.refreshMaterializedView(this.materializedViewNameLatest);
|
|
23
25
|
});
|
|
@@ -26,8 +28,30 @@ class EventsRepository extends integration_engine_1.PostgresModel {
|
|
|
26
28
|
});
|
|
27
29
|
this.refreshMaterializedView = (viewName) => __awaiter(this, void 0, void 0, function* () {
|
|
28
30
|
helpers_1.log.debug(`Refreshing materialized view: ${viewName}`);
|
|
29
|
-
const
|
|
30
|
-
yield
|
|
31
|
+
const semaphore = this.createSemaphore(this.materializedViewNameLatest);
|
|
32
|
+
const lockAcquired = yield semaphore.tryAcquire();
|
|
33
|
+
if (!lockAcquired) {
|
|
34
|
+
helpers_1.log.info(`Rush Hour Aggregation - unable to obtain mutex lock for: ${viewName}`);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
const sql = `REFRESH MATERIALIZED VIEW CONCURRENTLY ${PrahaDopravniInfo_1.default.SCHEMA}.${viewName}`;
|
|
39
|
+
yield this.query(sql);
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
yield semaphore.release();
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
createSemaphore(keyPhrase) {
|
|
47
|
+
const redisClient = integration_engine_1.RedisConnector.getConnection();
|
|
48
|
+
return new redis_semaphore_1.Mutex(redisClient, `RushHourAggregation-${keyPhrase}`, {
|
|
49
|
+
acquireAttemptsLimit: 1,
|
|
50
|
+
lockTimeout: this.lockTimeout,
|
|
51
|
+
refreshInterval: this.lockTimeout * 0.8,
|
|
52
|
+
onLockLost: (err) => {
|
|
53
|
+
helpers_1.log.warn(`RushHourAggregation - ${err.message}`);
|
|
54
|
+
},
|
|
31
55
|
});
|
|
32
56
|
}
|
|
33
57
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractEventsRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/AbstractEventsRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,oHAAwE;AACxE,
|
|
1
|
+
{"version":3,"file":"AbstractEventsRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/AbstractEventsRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,oHAAwE;AACxE,8EAA8F;AAC9F,2EAAoE;AACpE,+EAAkE;AAElE,MAA8B,gBAAoB,SAAQ,kCAAa;IAAvE;;QACqB,gBAAW,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;QAOtD,kCAA6B,GAAG,GAAwB,EAAE;YAC7D,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxE,CAAC,CAAA,CAAC;QAEK,oCAA+B,GAAG,GAAwB,EAAE;YAC/D,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1E,CAAC,CAAA,CAAC;QAEM,4BAAuB,GAAG,CAAO,QAAgB,EAAiB,EAAE;YACxE,aAAG,CAAC,KAAK,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAC;YAEvD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YACxE,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC;YAClD,IAAI,CAAC,YAAY,EAAE;gBACf,aAAG,CAAC,IAAI,CAAC,4DAA4D,QAAQ,EAAE,CAAC,CAAC;gBACjF,OAAO;aACV;YAED,IAAI;gBACA,MAAM,GAAG,GAAG,0CAA0C,2BAAiB,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAE7F,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;aACzB;oBAAS;gBACN,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;aAC7B;QACL,CAAC,CAAA,CAAC;IAaN,CAAC;IAXW,eAAe,CAAC,SAAiB;QACrC,MAAM,WAAW,GAAG,mCAAc,CAAC,aAAa,EAAE,CAAC;QACnD,OAAO,IAAI,uBAAK,CAAC,WAAW,EAAE,uBAAuB,SAAS,EAAE,EAAE;YAC9D,oBAAoB,EAAE,CAAC;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,eAAe,EAAE,IAAI,CAAC,WAAW,GAAG,GAAG;YACvC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChB,aAAG,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ;AA9CD,mCA8CC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { IDatabaseConnector } from "@golemio/core/dist/helpers/data-access/postgres/IDatabaseConnector";
|
|
2
|
+
import { AbstractBasicRepository } from "@golemio/core/dist/helpers/data-access/postgres/repositories/AbstractBasicRepository";
|
|
3
|
+
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
4
|
+
export declare class DataRetentionRepository extends AbstractBasicRepository {
|
|
5
|
+
schema: string;
|
|
6
|
+
tableName: string;
|
|
7
|
+
constructor(connector: IDatabaseConnector, log: ILogger);
|
|
8
|
+
deleteOldData(): Promise<void>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.DataRetentionRepository = void 0;
|
|
16
|
+
const PrahaDopravniInfo_1 = __importDefault(require("../../schema-definitions/sequelize-models/PrahaDopravniInfo"));
|
|
17
|
+
const AbstractBasicRepository_1 = require("@golemio/core/dist/helpers/data-access/postgres/repositories/AbstractBasicRepository");
|
|
18
|
+
class DataRetentionRepository extends AbstractBasicRepository_1.AbstractBasicRepository {
|
|
19
|
+
constructor(connector, log) {
|
|
20
|
+
super(connector, log);
|
|
21
|
+
this.schema = PrahaDopravniInfo_1.default.SCHEMA;
|
|
22
|
+
this.tableName = "";
|
|
23
|
+
}
|
|
24
|
+
deleteOldData() {
|
|
25
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
+
yield this.connector.getConnection().query(`CALL ${this.schema}.data_retention()`, {
|
|
27
|
+
plain: true,
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
exports.DataRetentionRepository = DataRetentionRepository;
|
|
33
|
+
//# sourceMappingURL=DataRetentionRepository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DataRetentionRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/dataAccess/DataRetentionRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,oHAAwE;AAExE,kIAA+H;AAG/H,MAAa,uBAAwB,SAAQ,iDAAuB;IAIhE,YAAY,SAA6B,EAAE,GAAY;QACnD,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAJnB,WAAM,GAAW,2BAAiB,CAAC,MAAM,CAAC;QAC1C,cAAS,GAAW,EAAE,CAAC;IAI9B,CAAC;IAEY,aAAa;;YACtB,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,MAAM,mBAAmB,EAAE;gBAC/E,KAAK,EAAE,IAAI;aACd,CAAC,CAAC;QACP,CAAC;KAAA;CACJ;AAbD,0DAaC"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import WazeReconstructionsRepository from "../../dataAccess/WazeReconstructionsRepository";
|
|
2
2
|
import WazeReconstructions from "../../../schema-definitions/sequelize-models/WazeReconstructions";
|
|
3
|
+
import { DurationLike } from "@golemio/core/dist/shared/luxon";
|
|
3
4
|
import AbstractAggregator from "./AbstractAggregator";
|
|
4
5
|
export default class WazeReconstructionsAggregator extends AbstractAggregator<WazeReconstructions> {
|
|
6
|
+
protected maxAggregationDuration: DurationLike;
|
|
5
7
|
protected repository: WazeReconstructionsRepository;
|
|
6
8
|
constructor();
|
|
7
9
|
protected aggregate: (from: Date, to: Date) => Promise<WazeReconstructions[]>;
|
|
@@ -21,6 +21,7 @@ const AbstractAggregator_1 = __importDefault(require("./AbstractAggregator"));
|
|
|
21
21
|
class WazeReconstructionsAggregator extends AbstractAggregator_1.default {
|
|
22
22
|
constructor() {
|
|
23
23
|
super();
|
|
24
|
+
this.maxAggregationDuration = { hours: 2 };
|
|
24
25
|
this.aggregate = (from, to) => __awaiter(this, void 0, void 0, function* () {
|
|
25
26
|
const result = yield AggregationRepository_1.default.getInstance().aggregationQuery(AggregationTaskType_1.AggregationTaskType.WAZER, {
|
|
26
27
|
from: from.getTime(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WazeReconstructionsAggregator.js","sourceRoot":"","sources":["../../../../src/integration-engine/service/aggregators/WazeReconstructionsAggregator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,mGAAyE;AACzE,mHAAyF;AACzF,iGAAuE;AACvE,wIAAyF;AACzF,2HAA4E;
|
|
1
|
+
{"version":3,"file":"WazeReconstructionsAggregator.js","sourceRoot":"","sources":["../../../../src/integration-engine/service/aggregators/WazeReconstructionsAggregator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,mGAAyE;AACzE,mHAAyF;AACzF,iGAAuE;AACvE,wIAAyF;AACzF,2HAA4E;AAE5E,8EAAsD;AAEtD,MAAqB,6BAA8B,SAAQ,4BAAuC;IAI9F;QACI,KAAK,EAAE,CAAC;QAJO,2BAAsB,GAAiB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAQ7D,cAAS,GAAG,CAAO,IAAU,EAAE,EAAQ,EAAkC,EAAE;YACjF,MAAM,MAAM,GAAG,MAAM,+BAAqB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CACrE,yCAAmB,CAAC,KAAK,EACzB;gBACI,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;gBACpB,EAAE,EAAE,EAAE,CAAC,OAAO,EAAE;aACnB,CACJ,CAAC;YAEF,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAA,CAAC;QAbE,IAAI,CAAC,UAAU,GAAG,IAAI,uCAA6B,EAAE,CAAC;IAC1D,CAAC;IAcS,QAAQ,CAAC,IAAW;QAC1B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACxB,OAAO;gBACH,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC;gBACjD,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,UAAU,EAAE,OAAO,CAAC,UAAU;aACV,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AAhCD,gDAgCC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/rush-hour-aggregation",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.6-dev.984077328",
|
|
4
4
|
"description": "Rush hour data aggregation for Praha dopravní project",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"@commitlint/cli": "^11.0.0",
|
|
33
33
|
"@commitlint/config-conventional": "^11.0.0",
|
|
34
34
|
"@golemio/cli": "1.4.3",
|
|
35
|
-
"@golemio/core": "1.8.
|
|
35
|
+
"@golemio/core": "1.8.10",
|
|
36
36
|
"@golemio/eslint-config": "1.1.1",
|
|
37
37
|
"@types/amqplib": "^0.5.17",
|
|
38
38
|
"@types/chai": "4.2.3",
|