@golemio/pid 2.7.13-dev.940112441 → 2.8.0

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.d.ts

@@ -1,14 +0,0 @@
-import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
-import { IDatasetsInput } from "./interfaces/IDatasetsInput";
-import { DatasetsInputSchema } from "./schema/DownloadDataInputSchema";
-export declare class CheckSavedRowsAndReplaceTablesTask extends AbstractTask<IDatasetsInput> {
-    readonly queueName = "checkSavedRowsAndReplaceTables";
-    readonly queueTtl: number;
-    readonly schema: typeof DatasetsInputSchema;
-    private logger;
-    private metaModel;
-    private dataCacheManager;
-    private precomputeTablesFacade;
-    constructor(queuePrefix: string);
-    protected execute(data: IDatasetsInput): Promise<void>;
-}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js

@@ -1,93 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CheckSavedRowsAndReplaceTablesTask = void 0;
-const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
-const DataCacheManager_1 = require("../../../helpers/DataCacheManager");
-const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
-const Di_1 = require("../../../ioc/Di");
-const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
-const constants_1 = require("../../../../vehicle-positions/workers/vehicle-positions/constants");
-const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
-const integration_engine_1 = require("@golemio/core/dist/integration-engine");
-const workers_1 = require("@golemio/core/dist/integration-engine/workers");
-const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
-const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
-class CheckSavedRowsAndReplaceTablesTask extends workers_1.AbstractTask {
-    constructor(queuePrefix) {
-        super(queuePrefix);
-        this.queueName = "checkSavedRowsAndReplaceTables";
-        this.queueTtl = 23 * 60 * 60 * 1000; // 23 hours
-        this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
-        this.logger = Di_1.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
-        // MetaData Model
-        this.metaModel = Di_1.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
-        this.dataCacheManager = DataCacheManager_1.DataCacheManager.getInstance();
-        this.precomputeTablesFacade = Di_1.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.PrecomputedTablesFacade);
-    }
-    execute(data) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const datasetDeployInfo = yield Promise.all(data.datasets.map((dataset) => __awaiter(this, void 0, void 0, function* () {
-                const lastModified = yield this.metaModel.getLastModified(dataset);
-                const isDeployed = yield this.metaModel.checkIfNewVersionIsAlreadyDeployed(dataset, lastModified.version);
-                return { dataset, lastModified, isDeployed };
-            })));
-            this.logger.debug(`Datasets status: ${JSON.stringify(datasetDeployInfo)}`);
-            const notDeployed = datasetDeployInfo.filter((info) => !info.isDeployed);
-            if (data.datasets.length !== notDeployed.length) {
-                const names = notDeployed.map((el) => el.dataset);
-                this.logger.error(`Error while synchronized download of datasets: [${names}] of [${data.datasets}] deployed already`);
-            }
-            if (!notDeployed.length) {
-                this.logger.debug(`Datasets status: already deployed`);
-                return;
-            }
-            const notDeployedVersions = notDeployed.map((el) => ({ dataset: el.dataset, version: el.lastModified.version }));
-            const allSaved = yield this.metaModel.checkAllTablesHasSavedState(notDeployedVersions);
-            try {
-                if (!allSaved) {
-                    throw new golemio_errors_1.GeneralError("Some GTFS datasets did not properly save", this.constructor.name);
-                }
-                yield Promise.all(datasetDeployInfo.map((el) => __awaiter(this, void 0, void 0, function* () {
-                    yield this.metaModel.checkSavedTmpTables(el.dataset, el.lastModified.version);
-                })));
-                yield this.precomputeTablesFacade.createAndPopulatePrecomputedTmpTables(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp);
-                yield this.metaModel.replaceTables(notDeployedVersions);
-                yield this.dataCacheManager.cleanCache();
-                yield integration_engine_1.QueueManager.sendMessageToExchange(constants_1.WORKER_NAME.toLowerCase(), "refreshGTFSTripData", {});
-                return;
-            }
-            catch (err) {
-                // log failed saving process
-                yield Promise.all(notDeployedVersions.map((el) => __awaiter(this, void 0, void 0, function* () {
-                    yield this.metaModel.rollbackFailedSaving(el.dataset, el.version);
-                })));
-                // check number of tries
-                const retries = yield Promise.all(notDeployedVersions.map((el) => __awaiter(this, void 0, void 0, function* () {
-                    return yield this.metaModel.getNumberOfDownloadRetries(el.dataset, el.version);
-                })));
-                if (Math.max(...retries) <= 5) {
-                    // send new downloadFiles, log it and finish
-                    integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "downloadDatasets", data);
-                    integration_engine_1.IntegrationErrorHandler.handle(new golemio_errors_1.GeneralError(`Error while checking RopidGTFS saved rows. Attempt number ${Math.min(...retries)} was resent.`, this.constructor.name, err));
-                    return;
-                }
-                else {
-                    // at least refresh precomputed tables and finish with error
-                    integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
-                    throw new golemio_errors_1.GeneralError("Error while checking RopidGTFS saved rows.", this.constructor.name, err);
-                }
-            }
-        });
-    }
-}
-exports.CheckSavedRowsAndReplaceTablesTask = CheckSavedRowsAndReplaceTablesTask;
-//# sourceMappingURL=CheckSavedRowsAndReplaceTablesTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"CheckSavedRowsAndReplaceTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+EAAoE;AAEpE,wEAA2E;AAC3E,kFAAqF;AACrF,wCAA2D;AAC3D,kFAAqF;AACrF,iGAA0G;AAC1G,wEAAqE;AAErE,8EAA8F;AAC9F,2EAA6E;AAC7E,6EAAwE;AAGxE,8EAAuE;AAEvE,MAAa,kCAAmC,SAAQ,sBAA4B;IAUhF,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QAVP,cAAS,GAAG,gCAAgC,CAAC;QAC7C,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAC3C,WAAM,GAAG,6CAAmB,CAAC;QASzC,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,iBAAiB;QACjB,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;QACjH,IAAI,CAAC,gBAAgB,GAAG,mCAAgB,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,CAAC,sBAAsB,GAAG,uBAAkB,CAAC,OAAO,CACpD,iDAAuB,CAAC,uBAAuB,CAClD,CAAC;IACN,CAAC;IAEe,OAAO,CAAC,IAAoB;;YACxC,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAO,OAAO,EAAE,EAAE;gBAChC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACnE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,kCAAkC,CAAC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;gBAC1G,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;YACjD,CAAC,CAAA,CAAC,CACL,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE3E,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,EAAE;gBAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,KAAK,SAAS,IAAI,CAAC,QAAQ,oBAAoB,CAAC,CAAC;aACzH;YAED,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBACvD,OAAO;aACV;YAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,CAAC;YAEvF,IAAI;gBACA,IAAI,CAAC,QAAQ,EAAE;oBACX,MAAM,IAAI,6BAAY,CAAC,0CAA0C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAC7F;gBAED,MAAM,OAAO,CAAC,GAAG,CACb,iBAAiB,CAAC,GAAG,CAAC,CAAO,EAAE,EAAE,EAAE;oBAC/B,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBAClF,CAAC,CAAA,CAAC,CACL,CAAC;gBAEF,MAAM,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,6CAAqB,CAAC,GAAG,CAAC,CAAC;gBACnG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;gBACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;gBACzC,MAAM,iCAAY,CAAC,qBAAqB,CAAC,uBAAc,CAAC,WAAW,EAAE,EAAE,qBAAqB,EAAE,EAAE,CAAC,CAAC;gBAElG,OAAO;aACV;YAAC,OAAO,GAAG,EAAE;gBACV,4BAA4B;gBAC5B,MAAM,OAAO,CAAC,GAAG,CACb,mBAAmB,CAAC,GAAG,CAAC,CAAO,EAAE,EAAE,EAAE;oBACjC,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;gBACtE,CAAC,CAAA,CAAC,CACL,CAAC;gBACF,wBAAwB;gBACxB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,mBAAmB,CAAC,GAAG,CAAC,CAAO,EAAE,EAAE,EAAE;oBACjC,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;gBACnF,CAAC,CAAA,CAAC,CACL,CAAC;gBAEF,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE;oBAC3B,4CAA4C;oBAC5C,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,EAAE,IAAI,CAAC,CAAC;oBAE/E,4CAAuB,CAAC,MAAM,CAC1B,IAAI,6BAAY,CACZ,6DAA6D,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,cAAc,EAC/F,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CACJ,CAAC;oBACF,OAAO;iBACV;qBAAM;oBACH,4DAA4D;oBAC5D,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;oBAErF,MAAM,IAAI,6BAAY,CAAC,4CAA4C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;iBACpG;aACJ;QACL,CAAC;KAAA;CACJ;AAlGD,gFAkGC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.d.ts

@@ -1,19 +0,0 @@
-import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
-import { IDatasetsInput } from "./interfaces/IDatasetsInput";
-import { DatasetsInputSchema } from "./schema/DownloadDataInputSchema";
-export declare class DownloadDatasetsTask extends AbstractTask<IDatasetsInput> {
-    readonly queueName = "downloadDatasets";
-    readonly queueTtl: number;
-    readonly schema: typeof DatasetsInputSchema;
-    private logger;
-    private metaModel;
-    private ropidGtfsFactory?;
-    private gtfsRedisChannel;
-    private staticFileRedisRepository;
-    constructor(queuePrefix: string);
-    protected execute(data: IDatasetsInput): Promise<void>;
-    /**
-     * Helper method - download and persist dataset
-     */
-    private downloadAndPersistDatasetFiles;
-}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.js

@@ -1,152 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DownloadDatasetsTask = void 0;
-const Di_1 = require("../../../../ioc/Di");
-const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
-const RopidGTFSMetadataModel_1 = require("../../../RopidGTFSMetadataModel");
-const Di_2 = require("../../../ioc/Di");
-const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
-const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
-const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
-const integration_engine_1 = require("@golemio/core/dist/integration-engine");
-const workers_1 = require("@golemio/core/dist/integration-engine/workers");
-const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
-const os_1 = __importDefault(require("os"));
-const HelperTypes_1 = require("./helpers/HelperTypes");
-const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
-class DownloadDatasetsTask extends workers_1.AbstractTask {
-    constructor(queuePrefix) {
-        super(queuePrefix);
-        this.queueName = "downloadDatasets";
-        this.queueTtl = 59 * 60 * 1000; // 59 minutes
-        this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
-        /**
-         * Helper method - download and persist dataset
-         */
-        this.downloadAndPersistDatasetFiles = (dataset) => __awaiter(this, void 0, void 0, function* () {
-            this.logger.info(`Datasets: processing ${dataset}`);
-            let files = yield this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getAll();
-            if (!Array.isArray(files))
-                files = [files];
-            const dbLastModified = yield this.metaModel.getLastModified(dataset);
-            const cachedDataset = [];
-            for (const file of files) {
-                this.logger.info(`Datasets: processing file '${file.name}'`);
-                const { data } = file, meta = __rest(file, ["data"]);
-                if (dataset === RopidGTFSMetadataModel_1.DatasetEnum.PID_GTFS || dataset === RopidGTFSMetadataModel_1.DatasetEnum.RUN_NUMBERS) {
-                    yield this.staticFileRedisRepository.pipeStream(file.filepath, data);
-                }
-                else {
-                    yield this.staticFileRedisRepository.set(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
-                }
-                const key = (0, HelperTypes_1.datasetFileModelMap)(dataset, file.name);
-                const metaStates = (Array.isArray(key) ? key : [key]).map((name) => {
-                    return {
-                        dataset,
-                        key: name,
-                        type: RopidMetadataModel_1.MetaTypeEnum.STATE,
-                        value: RopidMetadataModel_1.MetaStateEnum.DOWNLOADED,
-                        version: dbLastModified.version + 1,
-                    };
-                });
-                for (const metaState of metaStates) {
-                    yield this.metaModel.save(metaState);
-                }
-                cachedDataset.push(Object.assign(Object.assign({}, meta), { dataset }));
-            }
-            const lastModified = yield this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getLastModified();
-            yield this.metaModel.save({
-                dataset,
-                key: RopidMetadataModel_1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
-                type: RopidMetadataModel_1.MetaTypeEnum.DATASET_INFO,
-                value: lastModified,
-                version: dbLastModified.version + 1,
-            });
-            this.logger.info(`Datasets: processing ${dataset} finished - ${JSON.stringify(cachedDataset)}`);
-            return cachedDataset;
-        });
-        this.logger = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.Logger);
-        this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
-        this.staticFileRedisRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository);
-        // MetaData Model
-        this.metaModel = Di_2.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
-    }
-    execute(data) {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.logger.info(`Datasets: downloading ${data.datasets}`);
-            /** Ropid GTFS Dataset Map Dependencies **/
-            this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
-            let cachedDatasets = [];
-            for (const dataset of data.datasets) {
-                try {
-                    cachedDatasets = cachedDatasets.concat(yield this.downloadAndPersistDatasetFiles(dataset));
-                }
-                catch (err) {
-                    yield integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
-                    if (err instanceof golemio_errors_1.AbstractGolemioError) {
-                        throw err;
-                    }
-                    throw new golemio_errors_1.GeneralError("Error while downloading GTFS datasets", this.constructor.name, err);
-                }
-            }
-            const subscriber = this.gtfsRedisChannel.createSubscriber({ channelSuffix: os_1.default.hostname() });
-            yield subscriber.subscribe();
-            for (const cachedDataset of cachedDatasets) {
-                integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "transformAndSaveData", cachedDataset, {
-                    headers: {
-                        [HelperTypes_1.ORIGIN_HOSTNAME_HEADER]: os_1.default.hostname(),
-                    },
-                });
-            }
-            // Listen for messages from workers and wait for all of them to finish
-            // before continuing to the next step (replacing tables)
-            try {
-                yield subscriber.listen(cachedDatasets.length, ({ message, messageCount, isFinal }) => {
-                    this.logger.debug(`Datasets: received message: ${message} (${messageCount}/${cachedDatasets.length})`);
-                    if (!message.startsWith("OK")) {
-                        this.logger.error(`Datasets: subscription error: ${message}`);
-                        throw new golemio_errors_1.GeneralError(message, this.constructor.name);
-                    }
-                    if (isFinal) {
-                        this.logger.info("Datasets: all messages received");
-                    }
-                });
-            }
-            catch (err) {
-                if (err instanceof golemio_errors_1.AbstractGolemioError) {
-                    throw err;
-                }
-                throw new golemio_errors_1.GeneralError("Error while listening for messages", this.constructor.name, err);
-            }
-            finally {
-                yield subscriber.unsubscribe();
-                integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "checkSavedRowsAndReplaceTables", data);
-            }
-        });
-    }
-}
-exports.DownloadDatasetsTask = DownloadDatasetsTask;
-//# sourceMappingURL=DownloadDatasetsTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"DownloadDatasetsTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/DownloadDatasetsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA0C;AAC1C,+EAAoE;AACpE,4EAA4F;AAE5F,wCAA2D;AAC3D,kFAAqF;AACrF,8EAAoG;AACpG,wEAAqE;AAErE,8EAA4E;AAE5E,2EAA6E;AAC7E,6EAA8F;AAE9F,4CAAoB;AACpB,uDAAkH;AAGlH,8EAAuE;AAEvE,MAAa,oBAAqB,SAAQ,sBAA4B;IAYlE,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QAZP,cAAS,GAAG,kBAAkB,CAAC;QAC/B,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QACxC,WAAM,GAAG,6CAAmB,CAAC;QAgF7C;;WAEG;QACK,mCAA8B,GAAG,CAAO,OAAqB,EAA6B,EAAE;YAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;YACpD,IAAI,KAAK,GAAoB,MAAM,IAAI,CAAC,gBAAiB,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACvG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;YAE3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACrE,MAAM,aAAa,GAAqB,EAAE,CAAC;YAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC7D,MAAM,EAAE,IAAI,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAxB,QAAiB,CAAO,CAAC;gBAC/B,IAAI,OAAO,KAAK,oCAAW,CAAC,QAAQ,IAAI,OAAO,KAAK,oCAAW,CAAC,WAAW,EAAE;oBACzE,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAkB,CAAC,CAAC;iBACtF;qBAAM;oBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;iBACnH;gBAED,MAAM,GAAG,GAAG,IAAA,iCAAmB,EAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC/D,OAAO;wBACH,OAAO;wBACP,GAAG,EAAE,IAAI;wBACT,IAAI,EAAE,iCAAY,CAAC,KAAK;wBACxB,KAAK,EAAE,kCAAa,CAAC,UAAU;wBAC/B,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;qBACtC,CAAC;gBACN,CAAC,CAAC,CAAC;gBAEH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;oBAChC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBACxC;gBAED,aAAa,CAAC,IAAI,iCACX,IAAI,KACP,OAAO,IACT,CAAC;aACN;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAiB,CAAC,aAAa,EAAE,CAAC,OAAuB,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;YAExH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBACtB,OAAO;gBACP,GAAG,EAAE,2CAAsB,CAAC,aAAa;gBACzC,IAAI,EAAE,iCAAY,CAAC,YAAY;gBAC/B,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;aACtC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAChG,OAAO,aAAa,CAAC;QACzB,CAAC,CAAA,CAAC;QA1HE,IAAI,CAAC,MAAM,GAAG,iBAAY,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAqB,iDAAuB,CAAC,kBAAkB,CAAC,CAAC;QAEnH,IAAI,CAAC,yBAAyB,GAAG,uBAAkB,CAAC,OAAO,CACvD,iDAAuB,CAAC,yBAAyB,CACpD,CAAC;QAEF,iBAAiB;QACjB,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;IACrH,CAAC;IAEe,OAAO,CAAC,IAAoB;;YACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC3D,2CAA2C;YAC3C,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAmB,iDAAuB,CAAC,gBAAgB,CAAC,CAAC;YAE/G,IAAI,cAAc,GAAqB,EAAE,CAAC;YAE1C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACjC,IAAI;oBACA,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC,CAAC;iBAC9F;gBAAC,OAAO,GAAG,EAAE;oBACV,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;oBAE3F,IAAI,GAAG,YAAY,qCAAoB,EAAE;wBACrC,MAAM,GAAG,CAAC;qBACb;oBAED,MAAM,IAAI,6BAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;iBAC/F;aACJ;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,aAAa,EAAE,YAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC5F,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;YAE7B,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE;gBACxC,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE;oBACxF,OAAO,EAAE;wBACL,CAAC,oCAAsB,CAAC,EAAE,YAAE,CAAC,QAAQ,EAAE;qBAC1C;iBACJ,CAAC,CAAC;aACN;YAED,sEAAsE;YACtE,wDAAwD;YACxD,IAAI;gBACA,MAAM,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,EAAE;oBAClF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,OAAO,KAAK,YAAY,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;oBAEvG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;wBAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,OAAO,EAAE,CAAC,CAAC;wBAC9D,MAAM,IAAI,6BAAY,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;qBAC1D;oBAED,IAAI,OAAO,EAAE;wBACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;qBACvD;gBACL,CAAC,CAAC,CAAC;aACN;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,GAAG,YAAY,qCAAoB,EAAE;oBACrC,MAAM,GAAG,CAAC;iBACb;gBACD,MAAM,IAAI,6BAAY,CAAC,oCAAoC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC5F;oBAAS;gBACN,MAAM,UAAU,CAAC,WAAW,EAAE,CAAC;gBAC/B,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,gCAAgC,EAAE,IAAI,CAAC,CAAC;aAChG;QACL,CAAC;KAAA;CAwDJ;AAzID,oDAyIC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.d.ts

@@ -1,11 +0,0 @@
-import { AbstractEmptyTask } from "@golemio/core/dist/integration-engine/workers";
-export declare class RefreshPrecomputedTablesTask extends AbstractEmptyTask {
-    readonly queueName = "refreshPrecomputedTables";
-    readonly queueTtl: number;
-    private logger;
-    private metaModel;
-    private dataCacheManager;
-    private precomputeTablesFacade;
-    constructor(queuePrefix: string);
-    protected execute(): Promise<void>;
-}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js

@@ -1,57 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RefreshPrecomputedTablesTask = void 0;
-const Di_1 = require("../../../../ioc/Di");
-const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
-const DataCacheManager_1 = require("../../../helpers/DataCacheManager");
-const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
-const Di_2 = require("../../../ioc/Di");
-const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
-const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
-const workers_1 = require("@golemio/core/dist/integration-engine/workers");
-const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
-class RefreshPrecomputedTablesTask extends workers_1.AbstractEmptyTask {
-    constructor(queuePrefix) {
-        super(queuePrefix);
-        this.queueName = "refreshPrecomputedTables";
-        this.queueTtl = 59 * 60 * 1000; // 59 minutes
-        this.logger = Di_2.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
-        this.metaModel = Di_2.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
-        this.dataCacheManager = DataCacheManager_1.DataCacheManager.getInstance();
-        this.precomputeTablesFacade = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.PrecomputedTablesFacade);
-    }
-    execute() {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.precomputeTablesFacade.createAndPopulatePrecomputedTmpTables(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Actual);
-            }
-            catch (err) {
-                this.logger.error(err);
-                throw new golemio_errors_1.GeneralError("Error while creating RopidGTFS precomputed tables", this.constructor.name);
-            }
-            const connection = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.PostgresConnector).getConnection();
-            const transaction = yield connection.transaction();
-            try {
-                yield this.metaModel.replacePrecomputedTables(transaction);
-                yield transaction.commit();
-            }
-            catch (err) {
-                yield transaction.rollback();
-                this.logger.error(err);
-                throw new golemio_errors_1.GeneralError("Error while replacing RopidGTFS precomputed tables", this.constructor.name);
-            }
-            yield this.dataCacheManager.cleanCache();
-        });
-    }
-}
-exports.RefreshPrecomputedTablesTask = RefreshPrecomputedTablesTask;
-//# sourceMappingURL=RefreshPrecomputedTablesTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"RefreshPrecomputedTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA0C;AAC1C,+EAAoE;AAEpE,wEAA2E;AAC3E,kFAAqF;AACrF,wCAA2D;AAC3D,kFAAqF;AAErF,wEAAqE;AAErE,2EAAkF;AAClF,6EAAwE;AAGxE,MAAa,4BAA6B,SAAQ,2BAAiB;IAS/D,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QATP,cAAS,GAAG,0BAA0B,CAAC;QACvC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QASpD,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;QACjH,IAAI,CAAC,gBAAgB,GAAG,mCAAgB,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,CAAC,sBAAsB,GAAG,uBAAkB,CAAC,OAAO,CACpD,iDAAuB,CAAC,uBAAuB,CAClD,CAAC;IACN,CAAC;IAEe,OAAO;;YACnB,IAAI;gBACA,MAAM,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,6CAAqB,CAAC,MAAM,CAAC,CAAC;aACzG;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,MAAM,IAAI,6BAAY,CAAC,mDAAmD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACtG;YAED,MAAM,UAAU,GAAG,iBAAY,CAAC,OAAO,CAAqB,qBAAS,CAAC,iBAAiB,CAAC,CAAC,aAAa,EAAE,CAAC;YACzG,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,CAAC;YAEnD,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC;gBAC3D,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;aAC9B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;gBAE7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,MAAM,IAAI,6BAAY,CAAC,oDAAoD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACvG;YAED,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAC7C,CAAC;KAAA;CACJ;AA1CD,oEA0CC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.d.ts

@@ -1,21 +0,0 @@
-import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
-import { MessageProperties } from "amqplib";
-import { ICachedDataset } from "./helpers/HelperTypes";
-import { CachedDatasetSchema } from "./schema/CachedDatasetSchema";
-export declare class TransformAndSaveDataTask extends AbstractTask<ICachedDataset> {
-    readonly queueName = "transformAndSaveData";
-    readonly queueTtl: number;
-    readonly schema: typeof CachedDatasetSchema;
-    private logger;
-    private staticFileRedisRepository;
-    private transformationGtfs;
-    private ropidGtfsFactory;
-    private gtfsRedisChannel;
-    private metaModel;
-    constructor(queuePrefix: string);
-    protected execute(inputData: ICachedDataset, msgProperties?: MessageProperties): Promise<void>;
-    private saveTransformedDataset;
-    private createTmpModel;
-    private streamDataToTmp;
-    private getTmpModelByName;
-}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.js

@@ -1,160 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TransformAndSaveDataTask = void 0;
-const Di_1 = require("../../../../ioc/Di");
-const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
-const ropid_gtfs_1 = require("../../..");
-const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
-const Di_2 = require("../../../ioc/Di");
-const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
-const utils_1 = require("../../../utils");
-const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
-const const_1 = require("../../../../../schema-definitions/const");
-const ropid_gtfs_2 = require("../../../../../schema-definitions/ropid-gtfs");
-const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
-const PostgresModel_1 = require("@golemio/core/dist/integration-engine/models/PostgresModel");
-const workers_1 = require("@golemio/core/dist/integration-engine/workers");
-const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
-const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
-const pg_copy_streams_1 = require("pg-copy-streams");
-const HelperTypes_1 = require("./helpers/HelperTypes");
-const CachedDatasetSchema_1 = require("./schema/CachedDatasetSchema");
-class TransformAndSaveDataTask extends workers_1.AbstractTask {
-    constructor(queuePrefix) {
-        super(queuePrefix);
-        this.queueName = "transformAndSaveData";
-        this.queueTtl = 23 * 60 * 60 * 1000; // 23 hours
-        this.schema = CachedDatasetSchema_1.CachedDatasetSchema;
-        this.saveTransformedDataset = (transformedData, dataset, modelName) => __awaiter(this, void 0, void 0, function* () {
-            const model = yield this.createTmpModel(modelName);
-            switch (dataset) {
-                case ropid_gtfs_1.DatasetEnum.PID_GTFS:
-                case ropid_gtfs_1.DatasetEnum.RUN_NUMBERS:
-                    yield this.streamDataToTmp(transformedData, modelName);
-                    break;
-                default:
-                    yield (model === null || model === void 0 ? void 0 : model.save(transformedData, false));
-                    this.logger.info(`Datasets: copying to tmp success (${modelName})`);
-            }
-            const dbLastModified = yield this.metaModel.getLastModified(dataset);
-            yield this.metaModel.updateState(dataset, modelName, RopidMetadataModel_1.MetaStateEnum.SAVED, dbLastModified.version);
-        });
-        this.createTmpModel = (name) => __awaiter(this, void 0, void 0, function* () {
-            const model = yield this.getTmpModelByName(name);
-            if (ropid_gtfs_2.RopidGTFS[name].modelIndexOptions) {
-                yield model.query((0, utils_1.buildDropIndexQuery)(ropid_gtfs_2.RopidGTFS[name].modelIndexOptions));
-            }
-            yield model.sync({ force: true, schema: const_1.PG_SCHEMA });
-            return model;
-        });
-        this.streamDataToTmp = (data, modelName) => __awaiter(this, void 0, void 0, function* () {
-            // get connection and db client
-            const connection = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.PostgresConnector).getConnection();
-            const client = (yield connection.connectionManager.getConnection({ type: "write" }));
-            const tmpTableName = ropid_gtfs_2.RopidGTFS[modelName].pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp;
-            // copy transformed data to tmp table by stream
-            yield new Promise((resolve, reject) => {
-                const stream = client
-                    .query((0, pg_copy_streams_1.from)(`COPY "${const_1.PG_SCHEMA}".${tmpTableName} FROM STDIN DELIMITER ',' CSV HEADER;`))
-                    .on("error", (err) => __awaiter(this, void 0, void 0, function* () {
-                    this.logger.error(`Datasets: copying to tmp error (${modelName}): ${err.toString()}`);
-                    yield connection.connectionManager.releaseConnection(client);
-                    return reject(err);
-                }))
-                    .on("finish", () => __awaiter(this, void 0, void 0, function* () {
-                    this.logger.info(`Datasets: copying to tmp success (${modelName})`);
-                    yield connection.connectionManager.releaseConnection(client);
-                    return resolve();
-                }));
-                data.pipe(stream);
-            });
-        });
-        this.getTmpModelByName = (name) => __awaiter(this, void 0, void 0, function* () {
-            const modelDef = ropid_gtfs_2.RopidGTFS[name];
-            if (!modelDef || !modelDef.name || !modelDef.pgTableName || !modelDef.outputSequelizeAttributes) {
-                throw new Error("Model not found.");
-            }
-            const model = new PostgresModel_1.PostgresModel(modelDef.name + "Tmp" + "Model", {
-                hasTmpTable: false,
-                outputSequelizeAttributes: modelDef.outputSequelizeAttributes,
-                pgSchema: const_1.PG_SCHEMA,
-                pgTableName: modelDef.pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp,
-                savingType: "insertOnly",
-                sequelizeAdditionalSettings: Object.assign({ timestamps: false }, (modelDef.modelIndexOptions &&
-                    modelDef.modelIndexOptions.length && {
-                    indexes: modelDef.modelIndexOptions.map((indexDef) => {
-                        return Object.assign(Object.assign({}, indexDef), { name: indexDef.name + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp });
-                    }),
-                })),
-            }, new golemio_validator_1.JSONSchemaValidator(modelDef.name + "Tmp" + "ModelValidator", modelDef.outputMongooseSchemaObject));
-            return model;
-        });
-        this.logger = Di_2.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
-        this.metaModel = Di_2.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
-        this.staticFileRedisRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository);
-        this.transformationGtfs = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGTFSTransformation); // TODO is it posible to use factory instead, which already contain this transformation?
-        this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
-        this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
-    }
-    execute(inputData, msgProperties) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const datasetMap = this.ropidGtfsFactory.getDatasetMap();
-            let modelName = (0, HelperTypes_1.datasetFileModelMap)(inputData.dataset, inputData.name);
-            if (!Array.isArray(modelName)) {
-                modelName = [modelName];
-            }
-            let errorMessage;
-            if (inputData.dataset === ropid_gtfs_1.DatasetEnum.PID_GTFS || inputData.dataset === ropid_gtfs_1.DatasetEnum.RUN_NUMBERS) {
-                try {
-                    const sourceStream = this.staticFileRedisRepository.getReadableStream(inputData.filepath);
-                    const transformedDataset = yield this.transformationGtfs.transform({
-                        sourceStream,
-                        name: modelName[0],
-                    });
-                    yield this.saveTransformedDataset(transformedDataset, inputData.dataset, modelName[0]);
-                }
-                catch (err) {
-                    errorMessage = err.message;
-                }
-            }
-            else {
-                try {
-                    const data = yield this.staticFileRedisRepository.get(inputData.filepath);
-                    const transformedDataset = yield datasetMap[inputData.dataset].transformation.transform({
-                        data: datasetMap[inputData.dataset].cacheContentType === HelperTypes_1.CacheContentType.JSON ? JSON.parse(data) : data,
-                        name: inputData.name,
-                    });
-                    if (inputData.dataset === ropid_gtfs_1.DatasetEnum.CIS_STOPS) {
-                        yield this.saveTransformedDataset(transformedDataset.cis_stops, inputData.dataset, modelName[0]);
-                        yield this.saveTransformedDataset(transformedDataset.cis_stop_groups, inputData.dataset, modelName[1]);
-                    }
-                    else {
-                        yield this.saveTransformedDataset(transformedDataset, inputData.dataset, modelName[0]);
-                    }
-                }
-                catch (err) {
-                    errorMessage = err.message;
-                }
-            }
-            if (msgProperties === null || msgProperties === void 0 ? void 0 : msgProperties.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER]) {
-                yield this.gtfsRedisChannel.publishMessage(`${errorMessage !== null && errorMessage !== void 0 ? errorMessage : "OK"} (${modelName})`, {
-                    channelSuffix: msgProperties.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER],
-                });
-            }
-            if (errorMessage) {
-                throw new golemio_errors_1.GeneralError(errorMessage, this.constructor.name);
-            }
-        });
-    }
-}
-exports.TransformAndSaveDataTask = TransformAndSaveDataTask;
-//# sourceMappingURL=TransformAndSaveDataTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TransformAndSaveDataTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA0C;AAC1C,+EAAoE;AACpE,yCAA8F;AAE9F,kFAAqF;AACrF,wCAA2D;AAC3D,kFAAqF;AACrF,0CAA2D;AAC3D,8EAA8D;AAC9D,mEAAuC;AACvC,6EAAiF;AAGjF,wEAAqE;AAGrE,8FAA2F;AAC3F,2EAA6E;AAC7E,6EAAwE;AACxE,mFAAkF;AAElF,qDAAmD;AACnD,uDAM+B;AAE/B,sEAAmE;AAEnE,MAAa,wBAAyB,SAAQ,sBAA4B;IAYtE,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QAZP,cAAS,GAAG,sBAAsB,CAAC;QACnC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAC3C,WAAM,GAAG,yCAAmB,CAAC;QA0ErC,2BAAsB,GAAG,CAC7B,eAAoB,EACpB,OAAqB,EACrB,SAA2B,EACd,EAAE;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAEnD,QAAQ,OAAO,EAAE;gBACb,KAAK,wBAAW,CAAC,QAAQ,CAAC;gBAC1B,KAAK,wBAAW,CAAC,WAAW;oBACxB,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,SAA8B,CAAC,CAAC;oBAC5E,MAAM;gBACV;oBACI,MAAM,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAA,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,GAAG,CAAC,CAAC;aAC3E;YAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACrE,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE,kCAAa,CAAC,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;QACtG,CAAC,CAAA,CAAC;QAEM,mBAAc,GAAG,CAAO,IAAY,EAA0B,EAAE;YACpE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,sBAAS,CAAC,IAAuB,CAAC,CAAC,iBAAiB,EAAE;gBACtD,MAAM,KAAK,CAAC,KAAK,CAAC,IAAA,2BAAmB,EAAC,sBAAS,CAAC,IAAuB,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;aAChG;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAS,EAAE,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAA,CAAC;QAEM,oBAAe,GAAG,CAAO,IAAS,EAAE,SAA4B,EAAE,EAAE;YACxE,+BAA+B;YAC/B,MAAM,UAAU,GAAG,iBAAY,CAAC,OAAO,CAAqB,qBAAS,CAAC,iBAAiB,CAAC,CAAC,aAAa,EAAE,CAAC;YACzG,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,iBAAiB,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAQ,CAAC;YAE5F,MAAM,YAAY,GAAG,sBAAS,CAAC,SAA4B,CAAC,CAAC,WAAW,GAAG,6CAAqB,CAAC,GAAG,CAAC;YAErG,+CAA+C;YAC/C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACxC,MAAM,MAAM,GAAG,MAAM;qBAChB,KAAK,CAAC,IAAA,sBAAQ,EAAC,SAAS,iBAAS,KAAK,YAAY,uCAAuC,CAAC,CAAC;qBAC3F,EAAE,CAAC,OAAO,EAAE,CAAO,GAAQ,EAAE,EAAE;oBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,SAAS,MAAM,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBACtF,MAAM,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC7D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAA,CAAC;qBACD,EAAE,CAAC,QAAQ,EAAE,GAAS,EAAE;oBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,GAAG,CAAC,CAAC;oBACpE,MAAM,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC7D,OAAO,OAAO,EAAE,CAAC;gBACrB,CAAC,CAAA,CAAC,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;QAEM,sBAAiB,GAAG,CAAO,IAAY,EAA0B,EAAE;YACvE,MAAM,QAAQ,GAAG,sBAAS,CAAC,IAAuB,CAAC,CAAC;YAEpD,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,yBAAyB,EAAE;gBAC7F,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;aACvC;YAED,MAAM,KAAK,GAAG,IAAI,6BAAa,CAC3B,QAAQ,CAAC,IAAI,GAAG,KAAK,GAAG,OAAO,EAC/B;gBACI,WAAW,EAAE,KAAK;gBAClB,yBAAyB,EAAE,QAAQ,CAAC,yBAAyB;gBAC7D,QAAQ,EAAE,iBAAS;gBACnB,WAAW,EAAE,QAAQ,CAAC,WAAW,GAAG,6CAAqB,CAAC,GAAG;gBAC7D,UAAU,EAAE,YAAY;gBACxB,2BAA2B,kBACvB,UAAU,EAAE,KAAK,IACd,CAAC,QAAQ,CAAC,iBAAiB;oBAC1B,QAAQ,CAAC,iBAAiB,CAAC,MAAM,IAAI;oBACjC,OAAO,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAa,EAAE,EAAE;wBACtD,uCACO,QAAQ,KACX,IAAI,EAAE,QAAQ,CAAC,IAAI,GAAG,6CAAqB,CAAC,GAAG,IACjD;oBACN,CAAC,CAAC;iBACL,CAAC,CACT;aACJ,EACD,IAAI,uCAAmB,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK,GAAG,gBAAgB,EAAE,QAAQ,CAAC,0BAA0B,CAAC,CACzG,CAAC;YAEF,OAAO,KAAK,CAAC;QACjB,CAAC,CAAA,CAAC;QAtJE,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;QAEjH,IAAI,CAAC,yBAAyB,GAAG,uBAAkB,CAAC,OAAO,CACvD,iDAAuB,CAAC,yBAAyB,CACpD,CAAC;QACF,IAAI,CAAC,kBAAkB,GAAG,uBAAkB,CAAC,OAAO,CAChD,iDAAuB,CAAC,uBAAuB,CAClD,CAAC,CAAC,wFAAwF;QAC3F,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAmB,iDAAuB,CAAC,gBAAgB,CAAC,CAAC;QAC/G,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAqB,iDAAuB,CAAC,kBAAkB,CAAC,CAAC;IACvH,CAAC;IAEe,OAAO,CAAC,SAAyB,EAAE,aAAiC;;YAChF,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC;YACzD,IAAI,SAAS,GAAG,IAAA,iCAAmB,EAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YACvE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC3B,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;aAC3B;YAED,IAAI,YAAgC,CAAC;YACrC,IAAI,SAAS,CAAC,OAAO,KAAK,wBAAW,CAAC,QAAQ,IAAI,SAAS,CAAC,OAAO,KAAK,wBAAW,CAAC,WAAW,EAAE;gBAC7F,IAAI;oBACA,MAAM,YAAY,GAAG,IAAI,CAAC,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;oBAC1F,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;wBAC/D,YAAY;wBACZ,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;qBACrB,CAAC,CAAC;oBAEH,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC1F;gBAAC,OAAO,GAAG,EAAE;oBACV,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC;iBAC9B;aACJ;iBAAM;gBACH,IAAI;oBACA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;oBAC1E,MAAM,kBAAkB,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,OAAuB,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC;wBACpG,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,gBAAgB,KAAK,8BAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;wBACxG,IAAI,EAAE,SAAS,CAAC,IAAI;qBACvB,CAAC,CAAC;oBAEH,IAAI,SAAS,CAAC,OAAO,KAAK,wBAAW,CAAC,SAAS,EAAE;wBAC7C,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;wBACjG,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,eAAe,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;qBAC1G;yBAAM;wBACH,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;qBAC1F;iBACJ;gBAAC,OAAO,GAAG,EAAE;oBACV,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC;iBAC9B;aACJ;YAED,IAAI,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,OAAO,CAAC,oCAAsB,CAAC,EAAE;gBAChD,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,GAAG,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,IAAI,KAAK,SAAS,GAAG,EAAE;oBACjF,aAAa,EAAE,aAAa,CAAC,OAAO,CAAC,oCAAsB,CAAC;iBAC/D,CAAC,CAAC;aACN;YAED,IAAI,YAAY,EAAE;gBACd,MAAM,IAAI,6BAAY,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAC/D;QACL,CAAC;KAAA;CA0FJ;AArKD,4DAqKC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/HelperTypes.d.ts

@@ -1,24 +0,0 @@
-import { DatasetEnum } from "../../../../RopidGTFSMetadataModel";
-import { DatasetModelName } from "../../../../../../schema-definitions/ropid-gtfs";
-import { IDataSource } from "@golemio/core/dist/integration-engine/datasources/IDataSource";
-import { IFile } from "@golemio/core/dist/integration-engine/datasources/protocol-strategy/IProtocolStrategy";
-import { ITransformation } from "@golemio/core/dist/integration-engine/transformations/ITransformation";
-export declare const ORIGIN_HOSTNAME_HEADER = "x-origin-hostname";
-export type ICachedFileMeta = Omit<IFile, "data">;
-export type ICachedDataset = ICachedFileMeta & {
-    dataset: GTFSDatasets;
-};
-export declare enum CacheContentType {
-    CSV = "CSV",
-    JSON = "JSON"
-}
-export type GTFSDatasets = Exclude<DatasetEnum, DatasetEnum.DEPARTURES_PRESETS>;
-export type DatasetMap = {
-    [key in GTFSDatasets]: {
-        datasource: IDataSource;
-        transformation: ITransformation;
-        cacheContentType: CacheContentType;
-    };
-};
-export declare const isCachedDataset: (obj: any) => obj is ICachedDataset;
-export declare const datasetFileModelMap: (dataset: GTFSDatasets, name: string) => DatasetModelName | DatasetModelName[];