@golemio/pid 2.0.10-dev.372675355 → 2.0.11-dev.373518907

package/dist/integration-engine/ropid-gtfs/RopidGTFSWorker.js

@@ -20,7 +20,7 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RopidGTFSWorker = void 0;
+exports.RopidGTFSWorker = exports.datasetFileModelMap = void 0;
 const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
 const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
 const config_1 = require("@golemio/core/dist/integration-engine/config");
@@ -34,9 +34,40 @@ const ropid_gtfs_1 = require("../../schema-definitions/ropid-gtfs");
 const ropid_departures_presets_1 = require("../../schema-definitions/ropid-departures-presets");
 const pg_copy_streams_1 = require("pg-copy-streams");
 const vehicle_positions_1 = require("../../schema-definitions/vehicle-positions");
-const isCachedFileMeta = (obj) => {
-    return obj instanceof Object && obj.filepath !== undefined && obj.name !== undefined && obj.data === undefined;
+const RopidGTFSOisTransformation_1 = require("./transformations/RopidGTFSOisTransformation");
+var CacheContentType;
+(function (CacheContentType) {
+    CacheContentType["CSV"] = "CSV";
+    CacheContentType["JSON"] = "JSON";
+})(CacheContentType || (CacheContentType = {}));
+const isCachedDataset = (obj) => {
+    return (obj instanceof Object &&
+        obj.filepath !== undefined &&
+        obj.name !== undefined &&
+        obj.dataset !== undefined &&
+        obj.data === undefined);
 };
+const datasetFileModelMap = (dataset, name) => {
+    let model;
+    switch (dataset) {
+        case _1.DatasetEnum.PID_GTFS:
+            model = name;
+            break;
+        case _1.DatasetEnum.RUN_NUMBERS:
+            model = "run_numbers";
+            break;
+        case _1.DatasetEnum.OIS_MAPPING:
+            model = "ois";
+            break;
+        case _1.DatasetEnum.CIS_STOPS:
+            model = ["cis_stops", "cis_stop_groups"];
+            break;
+        default:
+            throw new Error(`Error mapping dataset ${dataset} and source ${name}`);
+    }
+    return model;
+};
+exports.datasetFileModelMap = datasetFileModelMap;
 class RopidGTFSWorker extends workers_1.BaseWorker {
     constructor() {
         super();
@@ -44,138 +75,211 @@ class RopidGTFSWorker extends workers_1.BaseWorker {
          * Queue listener method - triggers entire flow
          */
         this.checkForNewData = () => __awaiter(this, void 0, void 0, function* () {
-            // checking PID_GTFS dataset
-            const serverLastModified = yield this.dataSource.getLastModified();
-            const dbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.PID_GTFS);
-            if (serverLastModified !== dbLastModified.lastModified) {
-                yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadFiles", "Just do it!");
+            const datasets = [];
+            for (const dataset of Object.keys(this.datasetMap)) {
+                const serverLastModified = this.datasetMap[dataset].datasource.getLastModified();
+                const dbLastModified = yield this.metaModel.getLastModified(dataset);
+                if (serverLastModified !== dbLastModified.lastModified) {
+                    datasets.push(dataset);
+                }
             }
-            // checking CIS_STOPS dataset
-            const CISserverLastModified = yield this.dataSourceCisStops.getLastModified();
-            const CISdbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.CIS_STOPS);
-            if (CISserverLastModified !== CISdbLastModified.lastModified) {
-                yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadCisStops", "Just do it!");
+            if (datasets.length && datasets.length !== Object.keys(this.datasetMap).length) {
+                helpers_1.log.error(`RopidGTFS datasets are out of sync`);
             }
-            // checking RUN_NUMBERS dataset
-            const runNumbersServerLastModified = yield this.dataSourceRunNumbers.getLastModified();
-            const runNumbersDbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.RUN_NUMBERS);
-            if (runNumbersServerLastModified !== runNumbersDbLastModified.lastModified) {
-                yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadRunNumbers", "Just do it!");
+            if (datasets.length) {
+                yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadDatasets", JSON.stringify(datasets));
+            }
+            else {
+                helpers_1.log.info(`RopidGTFS datasets are up to date.`);
             }
         });
         /**
-         * Queue listener method - triggers for new presets only
+         * Queue listener method - triggers dataset download
+         * @param {object} msg
+         * @param {Buffer} msg.content - array of dataset names, empty to force downloading all
          */
-        this.checkForNewDeparturesPresets = () => __awaiter(this, void 0, void 0, function* () {
-            // checking DEPARTURES_PRESETS dataset
-            const departuresPresetsServerLastModified = yield this.dataSourceDeparturesPresets.getLastModified();
-            const departuresPresetsDbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.DEPARTURES_PRESETS);
-            if (departuresPresetsServerLastModified !== departuresPresetsDbLastModified.lastModified) {
-                yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadDeparturesPresets", "Just do it!");
+        this.downloadDatasets = (msg) => __awaiter(this, void 0, void 0, function* () {
+            let datasets;
+            try {
+                const content = msg.content.toString();
+                if (content) {
+                    datasets = JSON.parse(content);
+                    this.validateDatasetNames(datasets);
+                }
+                else {
+                    // utility setup - force run all if message empty
+                    datasets = Object.keys(this.datasetMap);
+                }
+            }
+            catch (err) {
+                throw new golemio_errors_1.CustomError("Error while parsing input data", true, this.constructor.name, undefined, err);
             }
+            helpers_1.log.debug(`Downloading datasets: ${datasets}`);
+            let transformMessages = [];
+            for (const dataset of datasets) {
+                transformMessages = transformMessages.concat(yield this.downloadAndPersistDatasetFiles(dataset));
+            }
+            yield Promise.all(transformMessages.map((msg) => __awaiter(this, void 0, void 0, function* () {
+                return this.sendMessageToExchange("workers." + this.queuePrefix + ".transformAndSaveData", JSON.stringify(msg));
+            })));
+            /**
+             * TODO remove sleep here, add meta state processing to above loop
+             * for checkSavedRowsAndReplaceTables not to pass without action
+             **/
+            yield new Promise((done) => setTimeout(done, 5 * 60 * 1000)); // sleeps for 5 minutes
+            yield this.sendMessageToExchange("workers." + this.queuePrefix + ".checkSavedRowsAndReplaceTables", JSON.stringify(datasets));
         });
         /**
-         * Queue listener method - triggers processing of PID_GTFS dataset
+         * Helper method - download and persist dataset
          */
-        this.downloadFiles = () => __awaiter(this, void 0, void 0, function* () {
-            const files = yield this.dataSource.getAll();
-            const cachedFileMeta = yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () {
+        this.downloadAndPersistDatasetFiles = (dataset) => __awaiter(this, void 0, void 0, function* () {
+            helpers_1.log.debug(`Processing datasets: ${dataset}`);
+            let files = yield this.datasetMap[dataset].datasource.getAll();
+            if (!Array.isArray(files))
+                files = [files];
+            const dbLastModified = yield this.metaModel.getLastModified(dataset);
+            const cachedDataset = yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () {
+                helpers_1.log.debug(`Processing datasets file: ${JSON.stringify(file.name)}`);
                 const { data } = file, meta = __rest(file, ["data"]);
-                yield this.redisModel.hset(file.filepath, data);
-                return Object.assign({}, meta);
+                yield this.redisModel.hset(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
+                const key = exports.datasetFileModelMap(dataset, file.name);
+                const metaState = (Array.isArray(key) ? key : [key]).map((name) => {
+                    return {
+                        dataset,
+                        key: name,
+                        type: _1.MetaTypeEnum.STATE,
+                        value: _1.MetaStateEnum.DOWNLOADED,
+                        version: dbLastModified.version + 1,
+                    };
+                });
+                yield this.metaModel.save(metaState);
+                return Object.assign(Object.assign({}, meta), { dataset });
             })));
-            const lastModified = yield this.dataSource.getLastModified();
-            const dbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.PID_GTFS);
+            const lastModified = yield this.datasetMap[dataset].datasource.getLastModified();
             yield this.metaModel.save({
-                dataset: _1.DatasetEnum.PID_GTFS,
+                dataset,
                 key: _1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
                 type: _1.MetaTypeEnum.DATASET_INFO,
                 value: lastModified,
-                version: dbLastModified.version + 1 || 1,
+                version: dbLastModified.version + 1,
             });
-            // send messages for transformation
-            const promises = cachedFileMeta.map((file) => __awaiter(this, void 0, void 0, function* () {
-                // save meta
-                yield this.metaModel.save({
-                    dataset: _1.DatasetEnum.PID_GTFS,
-                    key: file.name,
-                    type: _1.MetaTypeEnum.STATE,
-                    value: _1.MetaStateEnum.DOWNLOADED,
-                    version: dbLastModified.version + 1 || 1,
-                });
-                return this.sendMessageToExchange("workers." + this.queuePrefix + ".transformAndSaveData", JSON.stringify(file));
-            }));
-            yield Promise.all(promises);
-            // sleep 5 minute send checkSavedRowsAndReplaceTables
-            yield new Promise((done) => setTimeout(done, 5 * 60 * 1000));
-            yield this.sendMessageToExchange("workers." + this.queuePrefix + ".checkSavedRowsAndReplaceTables", "Just do it!");
+            helpers_1.log.debug(`Processing datasets: ${dataset} finished: ${JSON.stringify(cachedDataset)}`);
+            return cachedDataset;
+        });
+        /**
+         * Queue listener method - triggers for new presets only
+         */
+        this.checkForNewDeparturesPresets = () => __awaiter(this, void 0, void 0, function* () {
+            // checking DEPARTURES_PRESETS dataset
+            const departuresPresetsServerLastModified = yield this.dataSourceDeparturesPresets.getLastModified();
+            const departuresPresetsDbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.DEPARTURES_PRESETS);
+            if (departuresPresetsServerLastModified !== departuresPresetsDbLastModified.lastModified) {
+                yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadDeparturesPresets", "Just do it!");
+            }
         });
         /**
-         * Queue listener method - transform and save PID_GTFS dataset to tmp
+         * Queue listener method - transform and save dataset item to tmp
          */
         this.transformAndSaveData = (msg) => __awaiter(this, void 0, void 0, function* () {
             let inputData;
             try {
                 inputData = JSON.parse(msg.content.toString());
-                if (!isCachedFileMeta(inputData)) {
+                if (!isCachedDataset(inputData)) {
                     throw new Error("Input data has invalid format.");
                 }
             }
             catch (err) {
                 throw new golemio_errors_1.CustomError("Error while parsing input data", true, this.constructor.name, undefined, err);
             }
-            inputData.data = yield this.redisModel.hget(inputData.filepath);
-            const transformedData = yield this.transformation.transform(inputData);
-            const model = this.getModelByName(transformedData.name);
+            let data = this.datasetMap[inputData.dataset].cacheContentType === CacheContentType.JSON
+                ? JSON.parse(yield this.redisModel.hget(inputData.filepath))
+                : yield this.redisModel.hget(inputData.filepath);
+            const transformedDataset = yield this.datasetMap[inputData.dataset].transformation.transform({
+                data,
+                name: inputData.name,
+            });
+            if (inputData.dataset === _1.DatasetEnum.CIS_STOPS) {
+                const modelName = exports.datasetFileModelMap(inputData.dataset, inputData.name);
+                yield this.saveTransformedDataset(transformedDataset.cis_stops, inputData.dataset, modelName[0]);
+                yield this.saveTransformedDataset(transformedDataset.cis_stop_groups, inputData.dataset, modelName[1]);
+            }
+            else {
+                const modelName = exports.datasetFileModelMap(inputData.dataset, inputData.name);
+                yield this.saveTransformedDataset(transformedDataset, inputData.dataset, modelName);
+            }
+        });
+        this.saveTransformedDataset = (transformedData, dataset, modelName) => __awaiter(this, void 0, void 0, function* () {
+            const model = this.getModelByName(modelName);
             yield (model === null || model === void 0 ? void 0 : model.truncate(true));
-            yield this.saveDataToTmp(transformedData);
-            // update meta
-            const dbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.PID_GTFS);
-            yield this.metaModel.updateState(_1.DatasetEnum.PID_GTFS, transformedData.name, _1.MetaStateEnum.SAVED, dbLastModified.version);
+            switch (dataset) {
+                case _1.DatasetEnum.PID_GTFS:
+                    yield this.streamDataToTmp(transformedData, modelName);
+                    break;
+                default:
+                    yield (model === null || model === void 0 ? void 0 : model.save(transformedData, true));
+                    helpers_1.log.debug(`Copying finished. (${modelName})`);
+            }
+            const dbLastModified = yield this.metaModel.getLastModified(dataset);
+            yield this.metaModel.updateState(dataset, modelName, _1.MetaStateEnum.SAVED, dbLastModified.version);
         });
         /**
          * Queue listener method - verify tmp tables and replace public
          */
-        this.checkSavedRowsAndReplaceTables = () => __awaiter(this, void 0, void 0, function* () {
-            const dbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.PID_GTFS);
-            const alreadyDeployed = yield this.metaModel.checkIfNewVersionIsAlreadyDeployed(_1.DatasetEnum.PID_GTFS, dbLastModified.version);
-            if (alreadyDeployed) {
+        this.checkSavedRowsAndReplaceTables = (msg) => __awaiter(this, void 0, void 0, function* () {
+            let datasets;
+            try {
+                datasets = JSON.parse(msg.content.toString());
+                this.validateDatasetNames(datasets);
+            }
+            catch (err) {
+                throw new golemio_errors_1.CustomError("Error while parsing input data", true, this.constructor.name, undefined, err);
+            }
+            const datasetDeployInfo = yield Promise.all(datasets.map((dataset) => __awaiter(this, void 0, void 0, function* () {
+                const lastModified = yield this.metaModel.getLastModified(dataset);
+                const isDeployed = yield this.metaModel.checkIfNewVersionIsAlreadyDeployed(dataset, lastModified.version);
+                return { dataset, lastModified, isDeployed };
+            })));
+            helpers_1.log.debug(`Datasets status: ${JSON.stringify(datasetDeployInfo)}`);
+            const notDeployed = datasetDeployInfo.filter((info) => !info.isDeployed);
+            if (datasets.length !== notDeployed.length) {
+                const names = notDeployed.map((el) => el.dataset);
+                helpers_1.log.error(`Error while synchronized download of datasets: [${names}] of [${datasets}] deployed already`);
+            }
+            if (!notDeployed.length) {
+                helpers_1.log.debug(`Datasets status: already deployed`);
                 return true;
             }
-            const allSaved = yield this.metaModel.checkAllTablesHasSavedState(_1.DatasetEnum.PID_GTFS, dbLastModified.version);
+            const notDeployedVersions = notDeployed.map((el) => ({ dataset: el.dataset, version: el.lastModified.version }));
+            const allSaved = yield this.metaModel.checkAllTablesHasSavedState(notDeployedVersions);
             try {
                 if (!allSaved) {
                     // wait to the saving process
-                    helpers_1.log.debug("checkSavedRowsAndReplaceTables - not ready yet");
+                    helpers_1.log.debug("Datasets status: not ready yet");
                     yield new Promise((done) => setTimeout(done, 5 * 60 * 1000)); // sleeps for 5 minutes
-                    yield this.sendMessageToExchange("workers." + this.queuePrefix + ".checkSavedRowsAndReplaceTables", "Just do it!");
+                    yield this.sendMessageToExchange("workers." + this.queuePrefix + ".checkSavedRowsAndReplaceTables", JSON.stringify(datasets));
                     helpers_1.log.debug("checkSavedRowsAndReplaceTables - new message was resent");
                     return true;
                 }
-                yield this.metaModel.replaceTables(_1.DatasetEnum.PID_GTFS, dbLastModified.version);
+                yield this.metaModel.replaceTables(notDeployedVersions);
                 yield this.redisModel.truncate();
                 yield this.delayComputationTripsModel.truncate();
                 yield this.runTripsModel.truncate();
                 yield this.metaModel.refreshMaterializedViews();
-                // save meta
-                yield this.metaModel.save({
-                    dataset: _1.DatasetEnum.PID_GTFS,
-                    key: _1.MetaDatasetInfoKeyEnum.DEPLOYED,
-                    type: _1.MetaTypeEnum.DATASET_INFO,
-                    value: "true",
-                    version: dbLastModified.version,
-                });
                 return true;
             }
             catch (err) {
                 // log failed saving process
-                yield this.metaModel.rollbackFailedSaving(_1.DatasetEnum.PID_GTFS, dbLastModified.version);
+                yield Promise.all(notDeployedVersions.map((el) => __awaiter(this, void 0, void 0, function* () {
+                    yield this.metaModel.rollbackFailedSaving(el.dataset, el.version);
+                })));
                 // check number of tries
-                const retry = yield this.metaModel.getNumberOfDownloadRetries(_1.DatasetEnum.PID_GTFS, dbLastModified.version);
-                if (retry <= 5) {
+                const retries = yield Promise.all(notDeployedVersions.map((el) => __awaiter(this, void 0, void 0, function* () {
+                    return yield this.metaModel.getNumberOfDownloadRetries(el.dataset, el.version);
+                })));
+                if (Math.min(...retries) <= 5) {
                     // send new downloadFiles, log it and finish
-                    yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadFiles", "Just do it!");
-                    helpers_1.IntegrationErrorHandler.handle(new golemio_errors_1.CustomError(`Error while checking RopidGTFS saved rows. Attempt number ${retry} was resent.`, true, this.constructor.name, 5004, err));
+                    yield this.sendMessageToExchange("workers." + this.queuePrefix + ".downloadDatasets", JSON.stringify(datasets));
+                    helpers_1.IntegrationErrorHandler.handle(new golemio_errors_1.CustomError(`Error while checking RopidGTFS saved rows. Attempt number ${Math.min(...retries)} was resent.`, true, this.constructor.name, 5004, err));
                     return true;
                 }
                 else {
@@ -184,123 +288,6 @@ class RopidGTFSWorker extends workers_1.BaseWorker {
                 }
             }
         });
-        /**
-         * Queue listener method - triggers processing of CIS_STOPS dataset
-         */
-        this.downloadCisStops = () => __awaiter(this, void 0, void 0, function* () {
-            const file = yield this.dataSourceCisStops.getAll();
-            const lastModified = yield this.dataSourceCisStops.getLastModified();
-            const dbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.CIS_STOPS);
-            yield this.metaModel.save({
-                dataset: _1.DatasetEnum.CIS_STOPS,
-                key: _1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
-                type: _1.MetaTypeEnum.DATASET_INFO,
-                value: lastModified,
-                version: dbLastModified.version + 1,
-            });
-            const transformedData = yield this.transformationCisStops.transform(file.data);
-            // duplicity checking
-            // TODO is it ok if all duplicate items are removed?
-            const uniqueMap = new Map();
-            const duplicateCisStopGroups = [];
-            transformedData.cis_stop_groups.forEach((item) => {
-                if (uniqueMap.has(item.cis)) {
-                    duplicateCisStopGroups.push(item.cis);
-                    uniqueMap.delete(item.cis);
-                }
-                else {
-                    uniqueMap.set(item.cis, item);
-                }
-            });
-            const uniqueCisStopGroups = [...uniqueMap.values()];
-            if (duplicateCisStopGroups.length > 0) {
-                helpers_1.log.warn("CIS_STOP_GROUPS contains duplicate cis. Items with this cis was removed: " +
-                    JSON.stringify(duplicateCisStopGroups));
-            }
-            // save meta
-            yield this.metaModel.save([
-                {
-                    dataset: _1.DatasetEnum.CIS_STOPS,
-                    key: "cis_stop_groups",
-                    type: _1.MetaTypeEnum.TABLE_TOTAL_COUNT,
-                    value: uniqueCisStopGroups.length,
-                    version: dbLastModified.version + 1,
-                },
-                {
-                    dataset: _1.DatasetEnum.CIS_STOPS,
-                    key: "cis_stops",
-                    type: _1.MetaTypeEnum.TABLE_TOTAL_COUNT,
-                    value: transformedData.cis_stops.length,
-                    version: dbLastModified.version + 1,
-                },
-            ]);
-            try {
-                yield this.cisStopGroupsModel.truncate(true);
-                yield this.cisStopGroupsModel.save(uniqueCisStopGroups, true);
-                yield this.metaModel.save({
-                    dataset: _1.DatasetEnum.CIS_STOPS,
-                    key: "cis_stop_groups",
-                    type: _1.MetaTypeEnum.STATE,
-                    value: _1.MetaStateEnum.SAVED,
-                    version: dbLastModified.version + 1 || 1,
-                });
-                yield this.cisStopsModel.truncate(true);
-                yield this.cisStopsModel.save(transformedData.cis_stops, true);
-                yield this.metaModel.save({
-                    dataset: _1.DatasetEnum.CIS_STOPS,
-                    key: "cis_stops",
-                    type: _1.MetaTypeEnum.STATE,
-                    value: _1.MetaStateEnum.SAVED,
-                    version: dbLastModified.version + 1 || 1,
-                });
-                yield this.metaModel.checkSavedRows(_1.DatasetEnum.CIS_STOPS, dbLastModified.version + 1);
-                yield this.metaModel.replaceTables(_1.DatasetEnum.CIS_STOPS, dbLastModified.version + 1);
-            }
-            catch (err) {
-                helpers_1.log.error(err);
-                yield this.metaModel.rollbackFailedSaving(_1.DatasetEnum.CIS_STOPS, dbLastModified.version + 1);
-            }
-        });
-        /**
-         * Queue listener method - triggers processing of RUN_NUMBERS dataset
-         */
-        this.downloadRunNumbers = () => __awaiter(this, void 0, void 0, function* () {
-            const file = yield this.dataSourceRunNumbers.getAll();
-            const lastModified = yield this.dataSourceRunNumbers.getLastModified();
-            const dbLastModified = yield this.metaModel.getLastModified(_1.DatasetEnum.RUN_NUMBERS);
-            yield this.metaModel.save({
-                dataset: _1.DatasetEnum.RUN_NUMBERS,
-                key: _1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
-                type: _1.MetaTypeEnum.DATASET_INFO,
-                value: lastModified,
-                version: dbLastModified.version + 1,
-            });
-            const transformedData = yield this.transformationRunNumbers.transform(file.data);
-            yield this.metaModel.save({
-                dataset: _1.DatasetEnum.RUN_NUMBERS,
-                key: "runNumbers",
-                type: _1.MetaTypeEnum.TABLE_TOTAL_COUNT,
-                value: transformedData.length,
-                version: dbLastModified.version + 1,
-            });
-            try {
-                yield this.runNumbersModel.truncate(true);
-                yield this.runNumbersModel.save(transformedData, true);
-                yield this.metaModel.save({
-                    dataset: _1.DatasetEnum.RUN_NUMBERS,
-                    key: "runNumbers",
-                    type: _1.MetaTypeEnum.STATE,
-                    value: _1.MetaStateEnum.SAVED,
-                    version: dbLastModified.version + 1 || 1,
-                });
-                yield this.metaModel.checkSavedRows(_1.DatasetEnum.RUN_NUMBERS, dbLastModified.version + 1);
-                yield this.metaModel.replaceTables(_1.DatasetEnum.RUN_NUMBERS, dbLastModified.version + 1);
-            }
-            catch (err) {
-                helpers_1.log.error(err);
-                yield this.metaModel.rollbackFailedSaving(_1.DatasetEnum.RUN_NUMBERS, dbLastModified.version + 1);
-            }
-        });
         /**
          * Queue listener method - triggers processing of DEPARTURES_PRESETS dataset
          */
@@ -331,21 +318,48 @@ class RopidGTFSWorker extends workers_1.BaseWorker {
                     key: "departuresPresets",
                     type: _1.MetaTypeEnum.STATE,
                     value: _1.MetaStateEnum.SAVED,
-                    version: dbLastModified.version + 1 || 1,
+                    version: dbLastModified.version + 1,
                 });
                 yield this.metaModel.checkSavedRows(_1.DatasetEnum.DEPARTURES_PRESETS, dbLastModified.version + 1);
-                yield this.metaModel.replaceTables(_1.DatasetEnum.DEPARTURES_PRESETS, dbLastModified.version + 1);
+                yield this.metaModel.replaceTables([
+                    { dataset: _1.DatasetEnum.DEPARTURES_PRESETS, version: dbLastModified.version + 1 },
+                ]);
             }
             catch (err) {
                 helpers_1.log.error(err);
                 yield this.metaModel.rollbackFailedSaving(_1.DatasetEnum.DEPARTURES_PRESETS, dbLastModified.version + 1);
             }
         });
-        this.saveDataToTmp = (data) => __awaiter(this, void 0, void 0, function* () {
+        /**
+         * Validate dataset list against dataset enum
+         *
+         * @param {string[]} names - names of datasets
+         */
+        this.validateDatasetNames = (names) => {
+            if (!Array.isArray(names)) {
+                throw new Error(`Error validating dataset list: must be an array`);
+            }
+            if (!names.length) {
+                throw new Error(`Error validating dataset list: got empty array`);
+            }
+            const invalidNames = names.filter((name) => !this.validateDatasetName(name));
+            if (invalidNames.length) {
+                throw new Error(`Error validating dataset list: "${invalidNames.toString()}" is invalid`);
+            }
+        };
+        /**
+         * Validate name against dataset enum
+         *
+         * @param {string} name - name of datasets
+         */
+        this.validateDatasetName = (name) => {
+            return Object.keys(this.datasetMap).includes(name);
+        };
+        this.streamDataToTmp = (data, modelName) => __awaiter(this, void 0, void 0, function* () {
             // get connection and db client
             const connection = connectors_1.PostgresConnector.getConnection();
             const client = (yield connection.connectionManager.getConnection({ type: "write" }));
-            const tmpTableName = "tmp." + ropid_gtfs_1.RopidGTFS[data.name].pgTableName;
+            const tmpTableName = "tmp." + ropid_gtfs_1.RopidGTFS[modelName].pgTableName;
             // copy transformed data to tmp table by stream
             yield new Promise((resolve, reject) => {
                 const stream = client
@@ -356,42 +370,25 @@ class RopidGTFSWorker extends workers_1.BaseWorker {
                     return reject(err);
                 }))
                     .on("finish", () => __awaiter(this, void 0, void 0, function* () {
-                    helpers_1.log.debug(`Copying finished. (${data.name})`);
+                    helpers_1.log.debug(`Copying finished. (${modelName})`);
                     yield connection.connectionManager.releaseConnection(client);
                     return resolve();
                 }));
-                data.data.pipe(stream);
+                data.pipe(stream);
             });
         });
         this.getModelByName = (name) => {
-            if (ropid_gtfs_1.RopidGTFS[name].name) {
-                return new models_1.PostgresModel(ropid_gtfs_1.RopidGTFS[name].name + "Model", {
-                    hasTmpTable: true,
-                    outputSequelizeAttributes: ropid_gtfs_1.RopidGTFS[name].outputSequelizeAttributes,
-                    pgTableName: ropid_gtfs_1.RopidGTFS[name].pgTableName,
-                    savingType: "insertOnly",
-                }, new golemio_validator_1.Validator(ropid_gtfs_1.RopidGTFS[name].name + "ModelValidator", ropid_gtfs_1.RopidGTFS[name].outputMongooseSchemaObject));
+            if (!ropid_gtfs_1.RopidGTFS[name].name) {
+                throw new Error("Model name not found.");
             }
-            return null;
+            return new models_1.PostgresModel(ropid_gtfs_1.RopidGTFS[name].name + "Model", {
+                hasTmpTable: true,
+                outputSequelizeAttributes: ropid_gtfs_1.RopidGTFS[name].outputSequelizeAttributes,
+                pgTableName: ropid_gtfs_1.RopidGTFS[name].pgTableName,
+                savingType: "insertOnly",
+            }, new golemio_validator_1.Validator(ropid_gtfs_1.RopidGTFS[name].name + "ModelValidator", ropid_gtfs_1.RopidGTFS[name].outputMongooseSchemaObject));
         };
-        this.dataSource = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.name + "DataSource", new datasources_1.FTPProtocolStrategy({
-            filename: config_1.config.datasources.RopidGTFSFilename,
-            targetType: datasources_1.FTPTargetType.COMPRESSED,
-            path: config_1.config.datasources.RopidGTFSPath,
-            url: config_1.config.datasources.RopidFTP,
-            whitelistedFiles: [
-                "agency.txt",
-                "calendar.txt",
-                "calendar_dates.txt",
-                "shapes.txt",
-                "stop_times.txt",
-                "stops.txt",
-                "routes.txt",
-                "trips.txt",
-            ],
-            encoding: "utf8",
-        }), new datasources_1.JSONDataTypeStrategy({ resultsPath: "" }), null);
-        this.transformation = new _1.RopidGTFSTransformation();
+        // Redis Models
         this.redisModel = new models_1.RedisModel(ropid_gtfs_1.RopidGTFS.name + "Model", {
             isKeyConstructedFromData: false,
             prefix: "files:ropid",
@@ -400,51 +397,14 @@ class RopidGTFSWorker extends workers_1.BaseWorker {
             isKeyConstructedFromData: false,
             prefix: vehicle_positions_1.VehiclePositions.runs_trips.namespacePrefix,
         });
-        this.metaModel = new _1.RopidGTFSMetadataModel();
-        const cisStopsTypeStrategy = new datasources_1.JSONDataTypeStrategy({ resultsPath: "stopGroups" });
-        cisStopsTypeStrategy.setFilter((item) => item.cis !== 0);
-        this.dataSourceCisStops = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.name + "CisStops", new datasources_1.FTPProtocolStrategy({
-            filename: config_1.config.datasources.RopidGTFSCisStopsFilename,
-            path: config_1.config.datasources.RopidGTFSCisStopsPath,
-            url: config_1.config.datasources.RopidFTP,
-            encoding: "utf8",
-        }), cisStopsTypeStrategy, null);
-        this.transformationCisStops = new _1.RopidGTFSCisStopsTransformation();
-        this.cisStopGroupsModel = new models_1.PostgresModel(ropid_gtfs_1.RopidGTFS.cis_stop_groups.name + "Model", {
-            hasTmpTable: true,
-            outputSequelizeAttributes: ropid_gtfs_1.RopidGTFS.cis_stop_groups.outputSequelizeAttributes,
-            pgTableName: ropid_gtfs_1.RopidGTFS.cis_stop_groups.pgTableName,
-            savingType: "insertOnly",
-        }, new golemio_validator_1.Validator(ropid_gtfs_1.RopidGTFS.cis_stop_groups.name + "ModelValidator", ropid_gtfs_1.RopidGTFS.cis_stop_groups.outputMongooseSchemaObject));
-        this.cisStopsModel = new models_1.PostgresModel(ropid_gtfs_1.RopidGTFS.cis_stops.name + "Model", {
-            hasTmpTable: true,
-            outputSequelizeAttributes: ropid_gtfs_1.RopidGTFS.cis_stops.outputSequelizeAttributes,
-            pgTableName: ropid_gtfs_1.RopidGTFS.cis_stops.pgTableName,
-            savingType: "insertOnly",
-        }, new golemio_validator_1.Validator(ropid_gtfs_1.RopidGTFS.cis_stops.name + "ModelValidator", ropid_gtfs_1.RopidGTFS.cis_stops.outputMongooseSchemaObject));
         this.delayComputationTripsModel = new models_1.RedisModel(ropid_gtfs_1.RopidGTFS.delayComputationTrips.name + "Model", {
             decodeDataAfterGet: JSON.parse,
             encodeDataBeforeSave: JSON.stringify,
             isKeyConstructedFromData: true,
             prefix: ropid_gtfs_1.RopidGTFS.delayComputationTrips.mongoCollectionName,
         }, new golemio_validator_1.Validator(ropid_gtfs_1.RopidGTFS.delayComputationTrips.name + "ModelValidator", ropid_gtfs_1.RopidGTFS.delayComputationTrips.outputMongooseSchemaObject));
-        // Ropid GTFS Run Numbers
-        this.dataSourceRunNumbers = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.runNumbers.name + "DataSource", new datasources_1.FTPProtocolStrategy({
-            filename: config_1.config.datasources.RopidGTFSRunNumbersFilename,
-            path: config_1.config.datasources.RopidGTFSRunNumbersPath,
-            url: config_1.config.datasources.RopidFTP,
-            encoding: "utf8",
-        }), new datasources_1.CSVDataTypeStrategy({
-            fastcsvParams: { delimiter: ",", headers: true },
-            subscribe: (json) => json,
-        }), new golemio_validator_1.JSONSchemaValidator(ropid_gtfs_1.RopidGTFS.runNumbers.name + "DataSource", ropid_gtfs_1.RopidGTFS.runNumbers.datasourceJsonSchema));
-        this.transformationRunNumbers = new _1.RopidGTFSRunNumbersTransformation();
-        this.runNumbersModel = new models_1.PostgresModel(ropid_gtfs_1.RopidGTFS.runNumbers.name + "Model", {
-            hasTmpTable: true,
-            outputSequelizeAttributes: ropid_gtfs_1.RopidGTFS.runNumbers.outputSequelizeAttributes,
-            pgTableName: ropid_gtfs_1.RopidGTFS.runNumbers.pgTableName,
-            savingType: "insertOnly",
-        }, new golemio_validator_1.JSONSchemaValidator(ropid_gtfs_1.RopidGTFS.runNumbers.name + "ModelValidator", ropid_gtfs_1.RopidGTFS.runNumbers.outputJsonSchema));
+        // MetaData Model
+        this.metaModel = new _1.RopidGTFSMetadataModel();
         // Ropid Departures Presets
         this.dataSourceDeparturesPresets = new datasources_1.DataSource(ropid_departures_presets_1.RopidDeparturesPresets.name + "DataSource", new datasources_1.FTPProtocolStrategy({
             filename: config_1.config.datasources.RopidDeparturesPresetsFilename,
@@ -459,6 +419,77 @@ class RopidGTFSWorker extends workers_1.BaseWorker {
             pgTableName: ropid_departures_presets_1.RopidDeparturesPresets.pgTableName,
             savingType: "insertOnly",
         }, new golemio_validator_1.JSONSchemaValidator(ropid_departures_presets_1.RopidDeparturesPresets.name + "ModelValidator", ropid_departures_presets_1.RopidDeparturesPresets.outputJsonSchema));
+        /** Ropid GTFS Dataset Map Dependencies **/
+        // Ropid GTFS
+        this.dataSourceGtfs = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.name + "DataSource", new datasources_1.FTPProtocolStrategy({
+            filename: config_1.config.datasources.RopidGTFSFilename,
+            targetType: datasources_1.FTPTargetType.COMPRESSED,
+            path: config_1.config.datasources.RopidGTFSPath,
+            url: config_1.config.datasources.RopidFTP,
+            whitelistedFiles: [
+                "agency.txt",
+                "calendar.txt",
+                "calendar_dates.txt",
+                "shapes.txt",
+                "stop_times.txt",
+                "stops.txt",
+                "routes.txt",
+                "trips.txt",
+            ],
+            encoding: "utf8",
+        }), new datasources_1.JSONDataTypeStrategy({ resultsPath: "" }), null);
+        this.transformationGtfs = new _1.RopidGTFSTransformation();
+        // Ropid GTFS Run Numbers Datasource
+        this.dataSourceRunNumbers = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.run_numbers.name + "DataSource", new datasources_1.FTPProtocolStrategy({
+            filename: config_1.config.datasources.RopidGTFSRunNumbersFilename,
+            path: config_1.config.datasources.RopidGTFSRunNumbersPath,
+            url: config_1.config.datasources.RopidFTP,
+            encoding: "utf8",
+        }), new datasources_1.CSVDataTypeStrategy({
+            fastcsvParams: { delimiter: ",", headers: true },
+            subscribe: (json) => json,
+        }), new golemio_validator_1.JSONSchemaValidator(ropid_gtfs_1.RopidGTFS.run_numbers.name + "DataSource", ropid_gtfs_1.RopidGTFS.run_numbers.datasourceJsonSchema));
+        this.transformationRunNumbers = new _1.RopidGTFSRunNumbersTransformation();
+        // Ropid GTFS CIS STOPS Datasource
+        const cisStopsTypeStrategy = new datasources_1.JSONDataTypeStrategy({ resultsPath: "stopGroups" });
+        cisStopsTypeStrategy.setFilter((item) => item.cis !== 0);
+        this.dataSourceCisStops = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.name + "CisStops", new datasources_1.FTPProtocolStrategy({
+            filename: config_1.config.datasources.RopidGTFSCisStopsFilename,
+            path: config_1.config.datasources.RopidGTFSCisStopsPath,
+            url: config_1.config.datasources.RopidFTP,
+            encoding: "utf8",
+        }), cisStopsTypeStrategy, null);
+        this.transformationCisStops = new _1.RopidGTFSCisStopsTransformation();
+        // Ropid GTFS Ois Datasource
+        this.dataSourceOisMapping = new datasources_1.DataSource(ropid_gtfs_1.RopidGTFS.name + "Ois", new datasources_1.FTPProtocolStrategy({
+            filename: config_1.config.datasources.RopidGTFSOisFilename,
+            path: config_1.config.datasources.RopidGTFSOisPath,
+            url: config_1.config.datasources.RopidFTP,
+            encoding: "utf8",
+        }), new datasources_1.JSONDataTypeStrategy({ resultsPath: "" }), new golemio_validator_1.JSONSchemaValidator(ropid_gtfs_1.RopidGTFS.ois.name + "DataSource", ropid_gtfs_1.RopidGTFS.ois.datasourceJsonSchema));
+        this.transformationOisMapping = new RopidGTFSOisTransformation_1.RopidGTFSOisMappingTransformation();
+        this.datasetMap = {
+            [_1.DatasetEnum.PID_GTFS]: {
+                datasource: this.dataSourceGtfs,
+                transformation: this.transformationGtfs,
+                cacheContentType: CacheContentType.CSV,
+            },
+            [_1.DatasetEnum.CIS_STOPS]: {
+                datasource: this.dataSourceCisStops,
+                transformation: this.transformationCisStops,
+                cacheContentType: CacheContentType.JSON,
+            },
+            [_1.DatasetEnum.OIS_MAPPING]: {
+                datasource: this.dataSourceOisMapping,
+                transformation: this.transformationOisMapping,
+                cacheContentType: CacheContentType.JSON,
+            },
+            [_1.DatasetEnum.RUN_NUMBERS]: {
+                datasource: this.dataSourceRunNumbers,
+                transformation: this.transformationRunNumbers,
+                cacheContentType: CacheContentType.JSON,
+            },
+        };
         this.queuePrefix = config_1.config.RABBIT_EXCHANGE_NAME + "." + ropid_gtfs_1.RopidGTFS.name.toLowerCase();
     }
 }