@golemio/pid 2.8.2 → 2.8.3-dev.945754540

package/dist/integration-engine/ropid-gtfs/data-access/cache/StaticFileRedisRepository.js

@@ -1,15 +1,29 @@
 "use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var StaticFileRedisRepository_1;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StaticFileRedisRepository = void 0;
 const RedisModel_1 = require("@golemio/core/dist/integration-engine/models/RedisModel");
-class StaticFileRedisRepository extends RedisModel_1.RedisModel {
+const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
+let StaticFileRedisRepository = exports.StaticFileRedisRepository = StaticFileRedisRepository_1 = class StaticFileRedisRepository extends RedisModel_1.RedisModel {
     constructor() {
         super("GTFSStaticFileRedisRepository", {
             isKeyConstructedFromData: false,
-            prefix: StaticFileRedisRepository.NAMESPACE_PREFIX,
+            prefix: StaticFileRedisRepository_1.NAMESPACE_PREFIX,
         });
     }
-}
-exports.StaticFileRedisRepository = StaticFileRedisRepository;
+};
 StaticFileRedisRepository.NAMESPACE_PREFIX = "files:gtfsStatic";
+exports.StaticFileRedisRepository = StaticFileRedisRepository = StaticFileRedisRepository_1 = __decorate([
+    (0, tsyringe_1.injectable)(),
+    __metadata("design:paramtypes", [])
+], StaticFileRedisRepository);
 //# sourceMappingURL=StaticFileRedisRepository.js.map

package/dist/integration-engine/ropid-gtfs/data-access/cache/StaticFileRedisRepository.js.map

@@ -1 +1 @@
-{"version":3,"file":"StaticFileRedisRepository.js","sourceRoot":"","sources":["../../../../../src/integration-engine/ropid-gtfs/data-access/cache/StaticFileRedisRepository.ts"],"names":[],"mappings":";;;AAAA,wFAAqF;AAErF,MAAa,yBAA0B,SAAQ,uBAAU;IAGrD;QACI,KAAK,CAAC,+BAA+B,EAAE;YACnC,wBAAwB,EAAE,KAAK;YAC/B,MAAM,EAAE,yBAAyB,CAAC,gBAAgB;SACrD,CAAC,CAAC;IACP,CAAC;;AARL,8DASC;AARiB,0CAAgB,GAAG,kBAAkB,CAAC"}
+{"version":3,"file":"StaticFileRedisRepository.js","sourceRoot":"","sources":["../../../../../src/integration-engine/ropid-gtfs/data-access/cache/StaticFileRedisRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,wFAAqF;AACrF,iEAAgE;AAGzD,IAAM,yBAAyB,qEAA/B,MAAM,yBAA0B,SAAQ,uBAAU;IAGrD;QACI,KAAK,CAAC,+BAA+B,EAAE;YACnC,wBAAwB,EAAE,KAAK;YAC/B,MAAM,EAAE,2BAAyB,CAAC,gBAAgB;SACrD,CAAC,CAAC;IACP,CAAC;;AAPa,0CAAgB,GAAG,kBAAkB,AAArB,CAAsB;oCAD3C,yBAAyB;IADrC,IAAA,qBAAU,GAAE;;GACA,yBAAyB,CASrC"}

package/dist/integration-engine/ropid-gtfs/ioc/Di.d.ts

@@ -0,0 +1,3 @@
+import { DependencyContainer } from "@golemio/core/dist/shared/tsyringe";
+declare const RopidGtfsContainer: DependencyContainer;
+export { RopidGtfsContainer };

package/dist/integration-engine/ropid-gtfs/ioc/Di.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RopidGtfsContainer = void 0;
+const Di_1 = require("../../ioc/Di");
+const ropid_gtfs_1 = require("../../../schema-definitions/ropid-gtfs");
+const pubsub_1 = require("@golemio/core/dist/integration-engine/data-access/pubsub");
+const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
+const StaticFileRedisRepository_1 = require("../data-access/cache/StaticFileRedisRepository");
+const RopidGTFSTransformation_1 = require("../transformations/RopidGTFSTransformation");
+const PrecomputedTablesFacade_1 = require("../workers/timetables/tasks/helpers/PrecomputedTablesFacade");
+const RopidGtfsFactory_1 = require("../workers/timetables/tasks/helpers/RopidGtfsFactory");
+const RopidGtfsContainerToken_1 = require("./RopidGtfsContainerToken");
+//#region Initialization
+const RopidGtfsContainer = Di_1.PidContainer.createChildContainer();
+exports.RopidGtfsContainer = RopidGtfsContainer;
+//#endregion
+//#region Datasources
+//#endregion
+//#region Repositories
+RopidGtfsContainer.registerSingleton(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository, StaticFileRedisRepository_1.StaticFileRedisRepository);
+//#endregion
+//#region Transformations
+RopidGtfsContainer.registerSingleton(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGTFSTransformation, RopidGTFSTransformation_1.RopidGTFSTransformation);
+//#endregion
+//#region Facade
+RopidGtfsContainer.registerSingleton(RopidGtfsContainerToken_1.RopidGtfsContainerToken.PrecomputedTablesFacade, PrecomputedTablesFacade_1.PrecomputedTablesFacade)
+    .register(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory, RopidGtfsFactory_1.RopidGtfsFactory, { lifecycle: tsyringe_1.Lifecycle.Transient })
+    .registerInstance(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel, new pubsub_1.RedisPubSubChannel(ropid_gtfs_1.RopidGTFS.name));
+//# sourceMappingURL=Di.js.map

package/dist/integration-engine/ropid-gtfs/ioc/Di.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../../src/integration-engine/ropid-gtfs/ioc/Di.ts"],"names":[],"mappings":";;;AAAA,qCAA0C;AAC1C,uEAA4C;AAC5C,qFAA8F;AAC9F,iEAAoF;AACpF,8FAA2F;AAC3F,wFAAqF;AACrF,yGAAsG;AACtG,2FAAwF;AACxF,uEAAoE;AAEpE,wBAAwB;AACxB,MAAM,kBAAkB,GAAwB,iBAAY,CAAC,oBAAoB,EAAE,CAAC;AAqB3E,gDAAkB;AApB3B,YAAY;AAEZ,qBAAqB;AAErB,YAAY;AAEZ,sBAAsB;AACtB,kBAAkB,CAAC,iBAAiB,CAAC,iDAAuB,CAAC,yBAAyB,EAAE,qDAAyB,CAAC,CAAC;AACnH,YAAY;AAEZ,yBAAyB;AACzB,kBAAkB,CAAC,iBAAiB,CAAC,iDAAuB,CAAC,uBAAuB,EAAE,iDAAuB,CAAC,CAAC;AAC/G,YAAY;AAEZ,gBAAgB;AAChB,kBAAkB,CAAC,iBAAiB,CAAC,iDAAuB,CAAC,uBAAuB,EAAE,iDAAuB,CAAC;KACzG,QAAQ,CAAC,iDAAuB,CAAC,gBAAgB,EAAE,mCAAgB,EAAE,EAAE,SAAS,EAAE,oBAAS,CAAC,SAAS,EAAE,CAAC;KACxG,gBAAgB,CAAC,iDAAuB,CAAC,kBAAkB,EAAE,IAAI,2BAAkB,CAAC,sBAAS,CAAC,IAAI,CAAC,CAAC,CAAC"}

package/dist/integration-engine/ropid-gtfs/ioc/RopidGtfsContainerToken.d.ts

@@ -0,0 +1,8 @@
+declare const RopidGtfsContainerToken: {
+    PrecomputedTablesFacade: symbol;
+    StaticFileRedisRepository: symbol;
+    RopidGTFSTransformation: symbol;
+    RopidGtfsFactory: symbol;
+    RedisPubSubChannel: symbol;
+};
+export { RopidGtfsContainerToken };

package/dist/integration-engine/ropid-gtfs/ioc/RopidGtfsContainerToken.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RopidGtfsContainerToken = void 0;
+const RopidGtfsContainerToken = {
+    PrecomputedTablesFacade: Symbol(),
+    StaticFileRedisRepository: Symbol(),
+    RopidGTFSTransformation: Symbol(),
+    RopidGtfsFactory: Symbol(),
+    RedisPubSubChannel: Symbol(),
+};
+exports.RopidGtfsContainerToken = RopidGtfsContainerToken;
+//# sourceMappingURL=RopidGtfsContainerToken.js.map

package/dist/integration-engine/ropid-gtfs/ioc/RopidGtfsContainerToken.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RopidGtfsContainerToken.js","sourceRoot":"","sources":["../../../../src/integration-engine/ropid-gtfs/ioc/RopidGtfsContainerToken.ts"],"names":[],"mappings":";;;AAAA,MAAM,uBAAuB,GAAG;IAC5B,uBAAuB,EAAE,MAAM,EAAE;IACjC,yBAAyB,EAAE,MAAM,EAAE;IACnC,uBAAuB,EAAE,MAAM,EAAE;IACjC,gBAAgB,EAAE,MAAM,EAAE;IAC1B,kBAAkB,EAAE,MAAM,EAAE;CAC/B,CAAC;AAEO,0DAAuB"}

package/dist/integration-engine/ropid-gtfs/transformations/RopidGTFSTransformation.js

@@ -1,4 +1,13 @@
 "use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -16,10 +25,11 @@ exports.RopidGTFSTransformation = void 0;
 const ropid_gtfs_1 = require("../../../schema-definitions/ropid-gtfs");
 const integration_engine_1 = require("@golemio/core/dist/integration-engine");
 const transformations_1 = require("@golemio/core/dist/integration-engine/transformations");
+const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
 const csv_parser_1 = __importDefault(require("csv-parser"));
 const csv_stringify_1 = __importDefault(require("csv-stringify"));
 const stream_1 = require("stream");
-class RopidGTFSTransformation extends transformations_1.BaseTransformation {
+let RopidGTFSTransformation = exports.RopidGTFSTransformation = class RopidGTFSTransformation extends transformations_1.BaseTransformation {
     constructor() {
         super();
         this.transform = (input) => __awaiter(this, void 0, void 0, function* () {
@@ -52,6 +62,9 @@ class RopidGTFSTransformation extends transformations_1.BaseTransformation {
         this.transformElement = () => { };
         this.name = ropid_gtfs_1.RopidGTFS.name;
     }
-}
-exports.RopidGTFSTransformation = RopidGTFSTransformation;
+};
+exports.RopidGTFSTransformation = RopidGTFSTransformation = __decorate([
+    (0, tsyringe_1.injectable)(),
+    __metadata("design:paramtypes", [])
+], RopidGTFSTransformation);
 //# sourceMappingURL=RopidGTFSTransformation.js.map

package/dist/integration-engine/ropid-gtfs/transformations/RopidGTFSTransformation.js.map

@@ -1 +1 @@
-{"version":3,"file":"RopidGTFSTransformation.js","sourceRoot":"","sources":["../../../../src/integration-engine/ropid-gtfs/transformations/RopidGTFSTransformation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,uEAA4C;AAE5C,8EAA4D;AAC5D,2FAA4G;AAC5G,4DAA6B;AAC7B,kEAAsC;AACtC,mCAAuD;AAOvD,MAAa,uBAAwB,SAAQ,oCAAkB;IAG3D;QACI,KAAK,EAAE,CAAC;QAIL,cAAS,GAAG,CAAO,KAAmC,EAAqB,EAAE;YAChF,iEAAiE;YACjE,MAAM,4BAA4B,GAAG,MAAM,CAAC,IAAI,CAAC,sBAAS,CAAC,KAAK,CAAC,IAAuB,CAAC,CAAC,yBAAyB,CAAC;iBAC/G,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;iBACnD,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,EAA0B,CAAC,CAAC;YAElF,MAAM,UAAU,GAAG,IAAI,kBAAS,CAAC;gBAC7B,UAAU,EAAE,IAAI;gBAChB,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;oBAChC,IAAI,CAAC,IAAI,iCAEF,4BAA4B,GAE5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;yBAChB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;yBACxE,MAAM,CAAC,CAAC,GAAQ,EAAE,GAAG,EAAE,EAAE;wBACtB,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;wBACtB,OAAO,GAAG,CAAC;oBACf,CAAC,EAAE,EAAE,CAAC,EACZ,CAAC;oBACH,QAAQ,EAAE,CAAC;gBACf,CAAC;aACJ,CAAC,CAAC;YAEH,OAAO,IAAA,iBAAQ,EAAC,KAAK,CAAC,YAAY,EAAE,IAAA,oBAAG,GAAE,EAAE,UAAU,EAAE,IAAA,uBAAS,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE;gBACxF,IAAI,GAAG,EAAE;oBACL,wBAAG,CAAC,KAAK,CAAC,mCAAmC,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC/E,MAAM,GAAG,CAAC;iBACb;qBAAM;oBACH,wBAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;iBAChE;YACL,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;QAEQ,qBAAgB,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;QArClC,IAAI,CAAC,IAAI,GAAG,sBAAS,CAAC,IAAI,CAAC;IAC/B,CAAC;CAqCJ;AA3CD,0DA2CC"}
+{"version":3,"file":"RopidGTFSTransformation.js","sourceRoot":"","sources":["../../../../src/integration-engine/ropid-gtfs/transformations/RopidGTFSTransformation.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,uEAA4C;AAE5C,8EAA4D;AAC5D,2FAA4G;AAC5G,iEAAgE;AAChE,4DAA6B;AAC7B,kEAAsC;AACtC,mCAAuD;AAQhD,IAAM,uBAAuB,qCAA7B,MAAM,uBAAwB,SAAQ,oCAAkB;IAG3D;QACI,KAAK,EAAE,CAAC;QAIL,cAAS,GAAG,CAAO,KAAmC,EAAqB,EAAE;YAChF,iEAAiE;YACjE,MAAM,4BAA4B,GAAG,MAAM,CAAC,IAAI,CAAC,sBAAS,CAAC,KAAK,CAAC,IAAuB,CAAC,CAAC,yBAAyB,CAAC;iBAC/G,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;iBACnD,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,EAA0B,CAAC,CAAC;YAElF,MAAM,UAAU,GAAG,IAAI,kBAAS,CAAC;gBAC7B,UAAU,EAAE,IAAI;gBAChB,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;oBAChC,IAAI,CAAC,IAAI,iCAEF,4BAA4B,GAE5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;yBAChB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;yBACxE,MAAM,CAAC,CAAC,GAAQ,EAAE,GAAG,EAAE,EAAE;wBACtB,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;wBACtB,OAAO,GAAG,CAAC;oBACf,CAAC,EAAE,EAAE,CAAC,EACZ,CAAC;oBACH,QAAQ,EAAE,CAAC;gBACf,CAAC;aACJ,CAAC,CAAC;YAEH,OAAO,IAAA,iBAAQ,EAAC,KAAK,CAAC,YAAY,EAAE,IAAA,oBAAG,GAAE,EAAE,UAAU,EAAE,IAAA,uBAAS,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE;gBACxF,IAAI,GAAG,EAAE;oBACL,wBAAG,CAAC,KAAK,CAAC,mCAAmC,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC/E,MAAM,GAAG,CAAC;iBACb;qBAAM;oBACH,wBAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;iBAChE;YACL,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;QAEQ,qBAAgB,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;QArClC,IAAI,CAAC,IAAI,GAAG,sBAAS,CAAC,IAAI,CAAC;IAC/B,CAAC;CAqCJ,CAAA;kCA3CY,uBAAuB;IADnC,IAAA,qBAAU,GAAE;;GACA,uBAAuB,CA2CnC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/TimetableWorker.d.ts

@@ -0,0 +1,5 @@
+import { AbstractWorker } from "@golemio/core/dist/integration-engine/workers";
+export declare class TimetableWorker extends AbstractWorker {
+    protected name: string;
+    constructor();
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/TimetableWorker.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TimetableWorker = void 0;
+const workers_1 = require("@golemio/core/dist/integration-engine/workers");
+const CheckForNewDataTask_1 = require("./tasks/CheckForNewDataTask");
+const CheckSavedRowsAndReplaceTablesTask_1 = require("./tasks/CheckSavedRowsAndReplaceTablesTask");
+const DownloadDatasetsTask_1 = require("./tasks/DownloadDatasetsTask");
+const RefreshPrecomputedTablesTask_1 = require("./tasks/RefreshPrecomputedTablesTask");
+const TransformAndSaveDataTask_1 = require("./tasks/TransformAndSaveDataTask");
+class TimetableWorker extends workers_1.AbstractWorker {
+    constructor() {
+        super();
+        this.name = "ropidgtfs";
+        // Register tasks
+        this.registerTask(new CheckForNewDataTask_1.CheckForNewDataTask(this.getQueuePrefix()));
+        this.registerTask(new CheckSavedRowsAndReplaceTablesTask_1.CheckSavedRowsAndReplaceTablesTask(this.getQueuePrefix()));
+        this.registerTask(new DownloadDatasetsTask_1.DownloadDatasetsTask(this.getQueuePrefix()));
+        this.registerTask(new RefreshPrecomputedTablesTask_1.RefreshPrecomputedTablesTask(this.getQueuePrefix()));
+        this.registerTask(new TransformAndSaveDataTask_1.TransformAndSaveDataTask(this.getQueuePrefix()));
+    }
+}
+exports.TimetableWorker = TimetableWorker;
+//# sourceMappingURL=TimetableWorker.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/TimetableWorker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TimetableWorker.js","sourceRoot":"","sources":["../../../../../src/integration-engine/ropid-gtfs/workers/timetables/TimetableWorker.ts"],"names":[],"mappings":";;;AAAA,2EAA+E;AAC/E,qEAAkE;AAClE,mGAAgG;AAChG,uEAAoE;AACpE,uFAAoF;AACpF,+EAA4E;AAE5E,MAAa,eAAgB,SAAQ,wBAAc;IAG/C;QACI,KAAK,EAAE,CAAC;QAHF,SAAI,GAAG,WAAW,CAAC;QAKzB,iBAAiB;QACjB,IAAI,CAAC,YAAY,CAAC,IAAI,yCAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,YAAY,CAAC,IAAI,uEAAkC,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,YAAY,CAAC,IAAI,2CAAoB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,YAAY,CAAC,IAAI,2DAA4B,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,YAAY,CAAC,IAAI,mDAAwB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC3E,CAAC;CACJ;AAbD,0CAaC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckForNewDataTask.d.ts

@@ -0,0 +1,12 @@
+import { ICheckForNewDataInput } from "./interfaces/ICheckForNewDataInput";
+import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
+import { CheckForNewDataMessageValidation } from "./schema/CheckForNewDataMessageValidation";
+export declare class CheckForNewDataTask extends AbstractTask<ICheckForNewDataInput> {
+    readonly queueName = "checkForNewData";
+    readonly queueTtl: number;
+    readonly schema: typeof CheckForNewDataMessageValidation;
+    private logger;
+    private metaModel;
+    constructor(queuePrefix: string);
+    protected execute(data: ICheckForNewDataInput): Promise<void>;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckForNewDataTask.js

@@ -0,0 +1,61 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CheckForNewDataTask = void 0;
+const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
+const Di_1 = require("../../../ioc/Di");
+const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
+const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
+const integration_engine_1 = require("@golemio/core/dist/integration-engine");
+const workers_1 = require("@golemio/core/dist/integration-engine/workers");
+const CheckForNewDataMessageValidation_1 = require("./schema/CheckForNewDataMessageValidation");
+class CheckForNewDataTask extends workers_1.AbstractTask {
+    constructor(queuePrefix) {
+        super(queuePrefix);
+        this.queueName = "checkForNewData";
+        this.queueTtl = 19 * 60 * 1000; // 19 minutes
+        this.schema = CheckForNewDataMessageValidation_1.CheckForNewDataMessageValidation;
+        this.logger = Di_1.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
+        // MetaData Model
+        this.metaModel = Di_1.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
+    }
+    execute(data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let forceRefresh = !!data.forceRefresh;
+            const datasets = [];
+            const ropidGtfsFactory = Di_1.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
+            const datasetMap = ropidGtfsFactory.getDatasetMap();
+            for (const dataset of Object.keys(datasetMap)) {
+                const serverLastModified = yield datasetMap[dataset].datasource.getLastModified();
+                const dbLastModified = yield this.metaModel.getLastModified(dataset);
+                const isDeployed = yield this.metaModel.isDeployed(dataset);
+                if (serverLastModified !== dbLastModified.lastModified || !isDeployed) {
+                    datasets.push(dataset);
+                }
+            }
+            if (datasets.length && datasets.length !== Object.keys(datasetMap).length) {
+                this.logger.error(`RopidGTFS datasets are out of sync`);
+            }
+            if (datasets.length) {
+                yield integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "downloadDatasets", {
+                    datasets: datasets,
+                });
+                return;
+            }
+            this.logger.info("RopidGTFS datasets are up to date");
+            if (forceRefresh) {
+                yield integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
+            }
+        });
+    }
+}
+exports.CheckForNewDataTask = CheckForNewDataTask;
+//# sourceMappingURL=CheckForNewDataTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckForNewDataTask.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CheckForNewDataTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckForNewDataTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+EAAoE;AAEpE,wCAA2D;AAC3D,kFAAqF;AAErF,wEAAqE;AAErE,8EAAqE;AACrE,2EAA6E;AAI7E,gGAA6F;AAE7F,MAAa,mBAAoB,SAAQ,sBAAmC;IAQxE,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QARP,cAAS,GAAG,iBAAiB,CAAC;QAC9B,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QACxC,WAAM,GAAG,mEAAgC,CAAC;QAOtD,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,iBAAiB;QACjB,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;IACrH,CAAC;IAEe,OAAO,CAAC,IAA2B;;YAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;YACvC,MAAM,QAAQ,GAAG,EAAE,CAAC;YACpB,MAAM,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAmB,iDAAuB,CAAC,gBAAgB,CAAC,CAAC;YAChH,MAAM,UAAU,GAAG,gBAAgB,CAAC,aAAa,EAAE,CAAC;YAEpD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAmB,EAAE;gBAC7D,MAAM,kBAAkB,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;gBAClF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACrE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAE5D,IAAI,kBAAkB,KAAK,cAAc,CAAC,YAAY,IAAI,CAAC,UAAU,EAAE;oBACnE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC1B;aACJ;YAED,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;gBACvE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;aAC3D;YAED,IAAI,QAAQ,CAAC,MAAM,EAAE;gBACjB,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,EAAE;oBAC3E,QAAQ,EAAE,QAAQ;iBACH,CAAC,CAAC;gBACrB,OAAO;aACV;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YACtD,IAAI,YAAY,EAAE;gBACd,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;aAC9F;QACL,CAAC;KAAA;CACJ;AA/CD,kDA+CC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.d.ts

@@ -0,0 +1,14 @@
+import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
+import { IDatasetsInput } from "./interfaces/IDatasetsInput";
+import { DatasetsInputSchema } from "./schema/DownloadDataInputSchema";
+export declare class CheckSavedRowsAndReplaceTablesTask extends AbstractTask<IDatasetsInput> {
+    readonly queueName = "checkSavedRowsAndReplaceTables";
+    readonly queueTtl: number;
+    readonly schema: typeof DatasetsInputSchema;
+    private logger;
+    private metaModel;
+    private dataCacheManager;
+    private precomputeTablesFacade;
+    constructor(queuePrefix: string);
+    protected execute(data: IDatasetsInput): Promise<void>;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js

@@ -0,0 +1,93 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CheckSavedRowsAndReplaceTablesTask = void 0;
+const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
+const DataCacheManager_1 = require("../../../helpers/DataCacheManager");
+const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
+const Di_1 = require("../../../ioc/Di");
+const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
+const constants_1 = require("../../../../vehicle-positions/workers/vehicle-positions/constants");
+const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
+const integration_engine_1 = require("@golemio/core/dist/integration-engine");
+const workers_1 = require("@golemio/core/dist/integration-engine/workers");
+const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
+const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
+class CheckSavedRowsAndReplaceTablesTask extends workers_1.AbstractTask {
+    constructor(queuePrefix) {
+        super(queuePrefix);
+        this.queueName = "checkSavedRowsAndReplaceTables";
+        this.queueTtl = 23 * 60 * 60 * 1000; // 23 hours
+        this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
+        this.logger = Di_1.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
+        // MetaData Model
+        this.metaModel = Di_1.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
+        this.dataCacheManager = DataCacheManager_1.DataCacheManager.getInstance();
+        this.precomputeTablesFacade = Di_1.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.PrecomputedTablesFacade);
+    }
+    execute(data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const datasetDeployInfo = yield Promise.all(data.datasets.map((dataset) => __awaiter(this, void 0, void 0, function* () {
+                const lastModified = yield this.metaModel.getLastModified(dataset);
+                const isDeployed = yield this.metaModel.checkIfNewVersionIsAlreadyDeployed(dataset, lastModified.version);
+                return { dataset, lastModified, isDeployed };
+            })));
+            this.logger.debug(`Datasets status: ${JSON.stringify(datasetDeployInfo)}`);
+            const notDeployed = datasetDeployInfo.filter((info) => !info.isDeployed);
+            if (data.datasets.length !== notDeployed.length) {
+                const names = notDeployed.map((el) => el.dataset);
+                this.logger.error(`Error while synchronized download of datasets: [${names}] of [${data.datasets}] deployed already`);
+            }
+            if (!notDeployed.length) {
+                this.logger.debug(`Datasets status: already deployed`);
+                return;
+            }
+            const notDeployedVersions = notDeployed.map((el) => ({ dataset: el.dataset, version: el.lastModified.version }));
+            const allSaved = yield this.metaModel.checkAllTablesHasSavedState(notDeployedVersions);
+            try {
+                if (!allSaved) {
+                    throw new golemio_errors_1.GeneralError("Some GTFS datasets did not properly save", this.constructor.name);
+                }
+                yield Promise.all(datasetDeployInfo.map((el) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.metaModel.checkSavedTmpTables(el.dataset, el.lastModified.version);
+                })));
+                yield this.precomputeTablesFacade.createAndPopulatePrecomputedTmpTables(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp);
+                yield this.metaModel.replaceTables(notDeployedVersions);
+                yield this.dataCacheManager.cleanCache();
+                yield integration_engine_1.QueueManager.sendMessageToExchange(constants_1.WORKER_NAME.toLowerCase(), "refreshGTFSTripData", {});
+                return;
+            }
+            catch (err) {
+                // log failed saving process
+                yield Promise.all(notDeployedVersions.map((el) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.metaModel.rollbackFailedSaving(el.dataset, el.version);
+                })));
+                // check number of tries
+                const retries = yield Promise.all(notDeployedVersions.map((el) => __awaiter(this, void 0, void 0, function* () {
+                    return yield this.metaModel.getNumberOfDownloadRetries(el.dataset, el.version);
+                })));
+                if (Math.max(...retries) <= 5) {
+                    // send new downloadFiles, log it and finish
+                    integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "downloadDatasets", data);
+                    integration_engine_1.IntegrationErrorHandler.handle(new golemio_errors_1.GeneralError(`Error while checking RopidGTFS saved rows. Attempt number ${Math.min(...retries)} was resent.`, this.constructor.name, err));
+                    return;
+                }
+                else {
+                    // at least refresh precomputed tables and finish with error
+                    integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
+                    throw new golemio_errors_1.GeneralError("Error while checking RopidGTFS saved rows.", this.constructor.name, err);
+                }
+            }
+        });
+    }
+}
+exports.CheckSavedRowsAndReplaceTablesTask = CheckSavedRowsAndReplaceTablesTask;
+//# sourceMappingURL=CheckSavedRowsAndReplaceTablesTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CheckSavedRowsAndReplaceTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+EAAoE;AAEpE,wEAA2E;AAC3E,kFAAqF;AACrF,wCAA2D;AAC3D,kFAAqF;AACrF,iGAA0G;AAC1G,wEAAqE;AAErE,8EAA8F;AAC9F,2EAA6E;AAC7E,6EAAwE;AAGxE,8EAAuE;AAEvE,MAAa,kCAAmC,SAAQ,sBAA4B;IAUhF,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QAVP,cAAS,GAAG,gCAAgC,CAAC;QAC7C,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAC3C,WAAM,GAAG,6CAAmB,CAAC;QASzC,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,iBAAiB;QACjB,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;QACjH,IAAI,CAAC,gBAAgB,GAAG,mCAAgB,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,CAAC,sBAAsB,GAAG,uBAAkB,CAAC,OAAO,CACpD,iDAAuB,CAAC,uBAAuB,CAClD,CAAC;IACN,CAAC;IAEe,OAAO,CAAC,IAAoB;;YACxC,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAO,OAAO,EAAE,EAAE;gBAChC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACnE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,kCAAkC,CAAC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;gBAC1G,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;YACjD,CAAC,CAAA,CAAC,CACL,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3E,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,EAAE;gBAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,KAAK,SAAS,IAAI,CAAC,QAAQ,oBAAoB,CAAC,CAAC;aACzH;YAED,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBACvD,OAAO;aACV;YAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,CAAC;YAEvF,IAAI;gBACA,IAAI,CAAC,QAAQ,EAAE;oBACX,MAAM,IAAI,6BAAY,CAAC,0CAA0C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAC7F;gBAED,MAAM,OAAO,CAAC,GAAG,CACb,iBAAiB,CAAC,GAAG,CAAC,CAAO,EAAE,EAAE,EAAE;oBAC/B,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBAClF,CAAC,CAAA,CAAC,CACL,CAAC;gBAEF,MAAM,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,6CAAqB,CAAC,GAAG,CAAC,CAAC;gBACnG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;gBACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;gBACzC,MAAM,iCAAY,CAAC,qBAAqB,CAAC,uBAAc,CAAC,WAAW,EAAE,EAAE,qBAAqB,EAAE,EAAE,CAAC,CAAC;gBAElG,OAAO;aACV;YAAC,OAAO,GAAG,EAAE;gBACV,4BAA4B;gBAC5B,MAAM,OAAO,CAAC,GAAG,CACb,mBAAmB,CAAC,GAAG,CAAC,CAAO,EAAE,EAAE,EAAE;oBACjC,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;gBACtE,CAAC,CAAA,CAAC,CACL,CAAC;gBACF,wBAAwB;gBACxB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,mBAAmB,CAAC,GAAG,CAAC,CAAO,EAAE,EAAE,EAAE;oBACjC,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;gBACnF,CAAC,CAAA,CAAC,CACL,CAAC;gBAEF,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE;oBAC3B,4CAA4C;oBAC5C,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,EAAE,IAAI,CAAC,CAAC;oBAE/E,4CAAuB,CAAC,MAAM,CAC1B,IAAI,6BAAY,CACZ,6DAA6D,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,cAAc,EAC/F,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CACJ,CAAC;oBACF,OAAO;iBACV;qBAAM;oBACH,4DAA4D;oBAC5D,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;oBAErF,MAAM,IAAI,6BAAY,CAAC,4CAA4C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;iBACpG;aACJ;QACL,CAAC;KAAA;CACJ;AAlGD,gFAkGC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.d.ts

@@ -0,0 +1,19 @@
+import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
+import { IDatasetsInput } from "./interfaces/IDatasetsInput";
+import { DatasetsInputSchema } from "./schema/DownloadDataInputSchema";
+export declare class DownloadDatasetsTask extends AbstractTask<IDatasetsInput> {
+    readonly queueName = "downloadDatasets";
+    readonly queueTtl: number;
+    readonly schema: typeof DatasetsInputSchema;
+    private logger;
+    private metaModel;
+    private ropidGtfsFactory?;
+    private gtfsRedisChannel;
+    private staticFileRedisRepository;
+    constructor(queuePrefix: string);
+    protected execute(data: IDatasetsInput): Promise<void>;
+    /**
+     * Helper method - download and persist dataset
+     */
+    private downloadAndPersistDatasetFiles;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.js

@@ -0,0 +1,152 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DownloadDatasetsTask = void 0;
+const Di_1 = require("../../../../ioc/Di");
+const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
+const RopidGTFSMetadataModel_1 = require("../../../RopidGTFSMetadataModel");
+const Di_2 = require("../../../ioc/Di");
+const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
+const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
+const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
+const integration_engine_1 = require("@golemio/core/dist/integration-engine");
+const workers_1 = require("@golemio/core/dist/integration-engine/workers");
+const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
+const os_1 = __importDefault(require("os"));
+const HelperTypes_1 = require("./helpers/HelperTypes");
+const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
+class DownloadDatasetsTask extends workers_1.AbstractTask {
+    constructor(queuePrefix) {
+        super(queuePrefix);
+        this.queueName = "downloadDatasets";
+        this.queueTtl = 59 * 60 * 1000; // 59 minutes
+        this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
+        /**
+         * Helper method - download and persist dataset
+         */
+        this.downloadAndPersistDatasetFiles = (dataset) => __awaiter(this, void 0, void 0, function* () {
+            this.logger.info(`Datasets: processing ${dataset}`);
+            let files = yield this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getAll();
+            if (!Array.isArray(files))
+                files = [files];
+            const dbLastModified = yield this.metaModel.getLastModified(dataset);
+            const cachedDataset = [];
+            for (const file of files) {
+                this.logger.info(`Datasets: processing file '${file.name}'`);
+                const { data } = file, meta = __rest(file, ["data"]);
+                if (dataset === RopidGTFSMetadataModel_1.DatasetEnum.PID_GTFS || dataset === RopidGTFSMetadataModel_1.DatasetEnum.RUN_NUMBERS) {
+                    yield this.staticFileRedisRepository.pipeStream(file.filepath, data);
+                }
+                else {
+                    yield this.staticFileRedisRepository.set(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
+                }
+                const key = (0, HelperTypes_1.datasetFileModelMap)(dataset, file.name);
+                const metaStates = (Array.isArray(key) ? key : [key]).map((name) => {
+                    return {
+                        dataset,
+                        key: name,
+                        type: RopidMetadataModel_1.MetaTypeEnum.STATE,
+                        value: RopidMetadataModel_1.MetaStateEnum.DOWNLOADED,
+                        version: dbLastModified.version + 1,
+                    };
+                });
+                for (const metaState of metaStates) {
+                    yield this.metaModel.save(metaState);
+                }
+                cachedDataset.push(Object.assign(Object.assign({}, meta), { dataset }));
+            }
+            const lastModified = yield this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getLastModified();
+            yield this.metaModel.save({
+                dataset,
+                key: RopidMetadataModel_1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
+                type: RopidMetadataModel_1.MetaTypeEnum.DATASET_INFO,
+                value: lastModified,
+                version: dbLastModified.version + 1,
+            });
+            this.logger.info(`Datasets: processing ${dataset} finished - ${JSON.stringify(cachedDataset)}`);
+            return cachedDataset;
+        });
+        this.logger = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.Logger);
+        this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
+        this.staticFileRedisRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository);
+        // MetaData Model
+        this.metaModel = Di_2.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
+    }
+    execute(data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.logger.info(`Datasets: downloading ${data.datasets}`);
+            /** Ropid GTFS Dataset Map Dependencies **/
+            this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
+            let cachedDatasets = [];
+            for (const dataset of data.datasets) {
+                try {
+                    cachedDatasets = cachedDatasets.concat(yield this.downloadAndPersistDatasetFiles(dataset));
+                }
+                catch (err) {
+                    yield integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
+                    if (err instanceof golemio_errors_1.AbstractGolemioError) {
+                        throw err;
+                    }
+                    throw new golemio_errors_1.GeneralError("Error while downloading GTFS datasets", this.constructor.name, err);
+                }
+            }
+            const subscriber = this.gtfsRedisChannel.createSubscriber({ channelSuffix: os_1.default.hostname() });
+            yield subscriber.subscribe();
+            for (const cachedDataset of cachedDatasets) {
+                integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "transformAndSaveData", cachedDataset, {
+                    headers: {
+                        [HelperTypes_1.ORIGIN_HOSTNAME_HEADER]: os_1.default.hostname(),
+                    },
+                });
+            }
+            // Listen for messages from workers and wait for all of them to finish
+            // before continuing to the next step (replacing tables)
+            try {
+                yield subscriber.listen(cachedDatasets.length, ({ message, messageCount, isFinal }) => {
+                    this.logger.debug(`Datasets: received message: ${message} (${messageCount}/${cachedDatasets.length})`);
+                    if (!message.startsWith("OK")) {
+                        this.logger.error(`Datasets: subscription error: ${message}`);
+                        throw new golemio_errors_1.GeneralError(message, this.constructor.name);
+                    }
+                    if (isFinal) {
+                        this.logger.info("Datasets: all messages received");
+                    }
+                });
+            }
+            catch (err) {
+                if (err instanceof golemio_errors_1.AbstractGolemioError) {
+                    throw err;
+                }
+                throw new golemio_errors_1.GeneralError("Error while listening for messages", this.constructor.name, err);
+            }
+            finally {
+                yield subscriber.unsubscribe();
+                integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "checkSavedRowsAndReplaceTables", data);
+            }
+        });
+    }
+}
+exports.DownloadDatasetsTask = DownloadDatasetsTask;
+//# sourceMappingURL=DownloadDatasetsTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DownloadDatasetsTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA0C;AAC1C,+EAAoE;AACpE,4EAA4F;AAE5F,wCAA2D;AAC3D,kFAAqF;AACrF,8EAAoG;AACpG,wEAAqE;AAErE,8EAA4E;AAE5E,2EAA6E;AAC7E,6EAA8F;AAE9F,4CAAoB;AACpB,uDAAkH;AAGlH,8EAAuE;AAEvE,MAAa,oBAAqB,SAAQ,sBAA4B;IAYlE,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QAZP,cAAS,GAAG,kBAAkB,CAAC;QAC/B,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QACxC,WAAM,GAAG,6CAAmB,CAAC;QAgF7C;;WAEG;QACK,mCAA8B,GAAG,CAAO,OAAqB,EAA6B,EAAE;YAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;YACpD,IAAI,KAAK,GAAoB,MAAM,IAAI,CAAC,gBAAiB,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACvG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;YAE3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACrE,MAAM,aAAa,GAAqB,EAAE,CAAC;YAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC7D,MAAM,EAAE,IAAI,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAxB,QAAiB,CAAO,CAAC;gBAC/B,IAAI,OAAO,KAAK,oCAAW,CAAC,QAAQ,IAAI,OAAO,KAAK,oCAAW,CAAC,WAAW,EAAE;oBACzE,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAkB,CAAC,CAAC;iBACtF;qBAAM;oBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;iBACnH;gBAED,MAAM,GAAG,GAAG,IAAA,iCAAmB,EAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC/D,OAAO;wBACH,OAAO;wBACP,GAAG,EAAE,IAAI;wBACT,IAAI,EAAE,iCAAY,CAAC,KAAK;wBACxB,KAAK,EAAE,kCAAa,CAAC,UAAU;wBAC/B,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;qBACtC,CAAC;gBACN,CAAC,CAAC,CAAC;gBAEH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;oBAChC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBACxC;gBAED,aAAa,CAAC,IAAI,iCACX,IAAI,KACP,OAAO,IACT,CAAC;aACN;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAiB,CAAC,aAAa,EAAE,CAAC,OAAuB,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;YAExH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBACtB,OAAO;gBACP,GAAG,EAAE,2CAAsB,CAAC,aAAa;gBACzC,IAAI,EAAE,iCAAY,CAAC,YAAY;gBAC/B,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;aACtC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAChG,OAAO,aAAa,CAAC;QACzB,CAAC,CAAA,CAAC;QA1HE,IAAI,CAAC,MAAM,GAAG,iBAAY,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAqB,iDAAuB,CAAC,kBAAkB,CAAC,CAAC;QAEnH,IAAI,CAAC,yBAAyB,GAAG,uBAAkB,CAAC,OAAO,CACvD,iDAAuB,CAAC,yBAAyB,CACpD,CAAC;QAEF,iBAAiB;QACjB,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;IACrH,CAAC;IAEe,OAAO,CAAC,IAAoB;;YACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC3D,2CAA2C;YAC3C,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAmB,iDAAuB,CAAC,gBAAgB,CAAC,CAAC;YAE/G,IAAI,cAAc,GAAqB,EAAE,CAAC;YAE1C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACjC,IAAI;oBACA,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC,CAAC;iBAC9F;gBAAC,OAAO,GAAG,EAAE;oBACV,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;oBAE3F,IAAI,GAAG,YAAY,qCAAoB,EAAE;wBACrC,MAAM,GAAG,CAAC;qBACb;oBAED,MAAM,IAAI,6BAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;iBAC/F;aACJ;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,aAAa,EAAE,YAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC5F,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;YAE7B,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE;gBACxC,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE;oBACxF,OAAO,EAAE;wBACL,CAAC,oCAAsB,CAAC,EAAE,YAAE,CAAC,QAAQ,EAAE;qBAC1C;iBACJ,CAAC,CAAC;aACN;YAED,sEAAsE;YACtE,wDAAwD;YACxD,IAAI;gBACA,MAAM,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,EAAE;oBAClF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,OAAO,KAAK,YAAY,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;oBAEvG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;wBAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,OAAO,EAAE,CAAC,CAAC;wBAC9D,MAAM,IAAI,6BAAY,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;qBAC1D;oBAED,IAAI,OAAO,EAAE;wBACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;qBACvD;gBACL,CAAC,CAAC,CAAC;aACN;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,GAAG,YAAY,qCAAoB,EAAE;oBACrC,MAAM,GAAG,CAAC;iBACb;gBACD,MAAM,IAAI,6BAAY,CAAC,oCAAoC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC5F;oBAAS;gBACN,MAAM,UAAU,CAAC,WAAW,EAAE,CAAC;gBAC/B,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,gCAAgC,EAAE,IAAI,CAAC,CAAC;aAChG;QACL,CAAC;KAAA;CAwDJ;AAzID,oDAyIC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.d.ts

@@ -0,0 +1,11 @@
+import { AbstractEmptyTask } from "@golemio/core/dist/integration-engine/workers";
+export declare class RefreshPrecomputedTablesTask extends AbstractEmptyTask {
+    readonly queueName = "refreshPrecomputedTables";
+    readonly queueTtl: number;
+    private logger;
+    private metaModel;
+    private dataCacheManager;
+    private precomputeTablesFacade;
+    constructor(queuePrefix: string);
+    protected execute(): Promise<void>;
+}