@golemio/pid 2.8.1 → 2.8.2-dev.943885572

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js

@@ -0,0 +1,57 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RefreshPrecomputedTablesTask = void 0;
+const Di_1 = require("../../../../ioc/Di");
+const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
+const DataCacheManager_1 = require("../../../helpers/DataCacheManager");
+const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
+const Di_2 = require("../../../ioc/Di");
+const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
+const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
+const workers_1 = require("@golemio/core/dist/integration-engine/workers");
+const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
+class RefreshPrecomputedTablesTask extends workers_1.AbstractEmptyTask {
+    constructor(queuePrefix) {
+        super(queuePrefix);
+        this.queueName = "refreshPrecomputedTables";
+        this.queueTtl = 59 * 60 * 1000; // 59 minutes
+        this.logger = Di_2.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
+        this.metaModel = Di_2.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
+        this.dataCacheManager = DataCacheManager_1.DataCacheManager.getInstance();
+        this.precomputeTablesFacade = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.PrecomputedTablesFacade);
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.precomputeTablesFacade.createAndPopulatePrecomputedTmpTables(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Actual);
+            }
+            catch (err) {
+                this.logger.error(err);
+                throw new golemio_errors_1.GeneralError("Error while creating RopidGTFS precomputed tables", this.constructor.name);
+            }
+            const connection = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.PostgresConnector).getConnection();
+            const transaction = yield connection.transaction();
+            try {
+                yield this.metaModel.replacePrecomputedTables(transaction);
+                yield transaction.commit();
+            }
+            catch (err) {
+                yield transaction.rollback();
+                this.logger.error(err);
+                throw new golemio_errors_1.GeneralError("Error while replacing RopidGTFS precomputed tables", this.constructor.name);
+            }
+            yield this.dataCacheManager.cleanCache();
+        });
+    }
+}
+exports.RefreshPrecomputedTablesTask = RefreshPrecomputedTablesTask;
+//# sourceMappingURL=RefreshPrecomputedTablesTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RefreshPrecomputedTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA0C;AAC1C,+EAAoE;AAEpE,wEAA2E;AAC3E,kFAAqF;AACrF,wCAA2D;AAC3D,kFAAqF;AAErF,wEAAqE;AAErE,2EAAkF;AAClF,6EAAwE;AAGxE,MAAa,4BAA6B,SAAQ,2BAAiB;IAS/D,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QATP,cAAS,GAAG,0BAA0B,CAAC;QACvC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QASpD,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;QACjH,IAAI,CAAC,gBAAgB,GAAG,mCAAgB,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,CAAC,sBAAsB,GAAG,uBAAkB,CAAC,OAAO,CACpD,iDAAuB,CAAC,uBAAuB,CAClD,CAAC;IACN,CAAC;IAEe,OAAO;;YACnB,IAAI;gBACA,MAAM,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,6CAAqB,CAAC,MAAM,CAAC,CAAC;aACzG;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,MAAM,IAAI,6BAAY,CAAC,mDAAmD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACtG;YAED,MAAM,UAAU,GAAG,iBAAY,CAAC,OAAO,CAAqB,qBAAS,CAAC,iBAAiB,CAAC,CAAC,aAAa,EAAE,CAAC;YACzG,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,CAAC;YAEnD,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC;gBAC3D,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;aAC9B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;gBAE7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,MAAM,IAAI,6BAAY,CAAC,oDAAoD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACvG;YAED,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAC7C,CAAC;KAAA;CACJ;AA1CD,oEA0CC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.d.ts

@@ -0,0 +1,21 @@
+import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
+import { MessageProperties } from "amqplib";
+import { ICachedDataset } from "./helpers/HelperTypes";
+import { CachedDatasetSchema } from "./schema/CachedDatasetSchema";
+export declare class TransformAndSaveDataTask extends AbstractTask<ICachedDataset> {
+    readonly queueName = "transformAndSaveData";
+    readonly queueTtl: number;
+    readonly schema: typeof CachedDatasetSchema;
+    private logger;
+    private staticFileRedisRepository;
+    private transformationGtfs;
+    private ropidGtfsFactory;
+    private gtfsRedisChannel;
+    private metaModel;
+    constructor(queuePrefix: string);
+    protected execute(inputData: ICachedDataset, msgProperties?: MessageProperties): Promise<void>;
+    private saveTransformedDataset;
+    private createTmpModel;
+    private streamDataToTmp;
+    private getTmpModelByName;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.js

@@ -0,0 +1,160 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformAndSaveDataTask = void 0;
+const Di_1 = require("../../../../ioc/Di");
+const ModuleContainerToken_1 = require("../../../../ioc/ModuleContainerToken");
+const ropid_gtfs_1 = require("../../..");
+const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
+const Di_2 = require("../../../ioc/Di");
+const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
+const utils_1 = require("../../../utils");
+const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
+const const_1 = require("../../../../../schema-definitions/const");
+const ropid_gtfs_2 = require("../../../../../schema-definitions/ropid-gtfs");
+const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
+const PostgresModel_1 = require("@golemio/core/dist/integration-engine/models/PostgresModel");
+const workers_1 = require("@golemio/core/dist/integration-engine/workers");
+const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
+const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
+const pg_copy_streams_1 = require("pg-copy-streams");
+const HelperTypes_1 = require("./helpers/HelperTypes");
+const CachedDatasetSchema_1 = require("./schema/CachedDatasetSchema");
+class TransformAndSaveDataTask extends workers_1.AbstractTask {
+    constructor(queuePrefix) {
+        super(queuePrefix);
+        this.queueName = "transformAndSaveData";
+        this.queueTtl = 23 * 60 * 60 * 1000; // 23 hours
+        this.schema = CachedDatasetSchema_1.CachedDatasetSchema;
+        this.saveTransformedDataset = (transformedData, dataset, modelName) => __awaiter(this, void 0, void 0, function* () {
+            const model = yield this.createTmpModel(modelName);
+            switch (dataset) {
+                case ropid_gtfs_1.DatasetEnum.PID_GTFS:
+                case ropid_gtfs_1.DatasetEnum.RUN_NUMBERS:
+                    yield this.streamDataToTmp(transformedData, modelName);
+                    break;
+                default:
+                    yield (model === null || model === void 0 ? void 0 : model.save(transformedData, false));
+                    this.logger.info(`Datasets: copying to tmp success (${modelName})`);
+            }
+            const dbLastModified = yield this.metaModel.getLastModified(dataset);
+            yield this.metaModel.updateState(dataset, modelName, RopidMetadataModel_1.MetaStateEnum.SAVED, dbLastModified.version);
+        });
+        this.createTmpModel = (name) => __awaiter(this, void 0, void 0, function* () {
+            const model = yield this.getTmpModelByName(name);
+            if (ropid_gtfs_2.RopidGTFS[name].modelIndexOptions) {
+                yield model.query((0, utils_1.buildDropIndexQuery)(ropid_gtfs_2.RopidGTFS[name].modelIndexOptions));
+            }
+            yield model.sync({ force: true, schema: const_1.PG_SCHEMA });
+            return model;
+        });
+        this.streamDataToTmp = (data, modelName) => __awaiter(this, void 0, void 0, function* () {
+            // get connection and db client
+            const connection = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.PostgresConnector).getConnection();
+            const client = (yield connection.connectionManager.getConnection({ type: "write" }));
+            const tmpTableName = ropid_gtfs_2.RopidGTFS[modelName].pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp;
+            // copy transformed data to tmp table by stream
+            yield new Promise((resolve, reject) => {
+                const stream = client
+                    .query((0, pg_copy_streams_1.from)(`COPY "${const_1.PG_SCHEMA}".${tmpTableName} FROM STDIN DELIMITER ',' CSV HEADER;`))
+                    .on("error", (err) => __awaiter(this, void 0, void 0, function* () {
+                    this.logger.error(`Datasets: copying to tmp error (${modelName}): ${err.toString()}`);
+                    yield connection.connectionManager.releaseConnection(client);
+                    return reject(err);
+                }))
+                    .on("finish", () => __awaiter(this, void 0, void 0, function* () {
+                    this.logger.info(`Datasets: copying to tmp success (${modelName})`);
+                    yield connection.connectionManager.releaseConnection(client);
+                    return resolve();
+                }));
+                data.pipe(stream);
+            });
+        });
+        this.getTmpModelByName = (name) => __awaiter(this, void 0, void 0, function* () {
+            const modelDef = ropid_gtfs_2.RopidGTFS[name];
+            if (!modelDef || !modelDef.name || !modelDef.pgTableName || !modelDef.outputSequelizeAttributes) {
+                throw new Error("Model not found.");
+            }
+            const model = new PostgresModel_1.PostgresModel(modelDef.name + "Tmp" + "Model", {
+                hasTmpTable: false,
+                outputSequelizeAttributes: modelDef.outputSequelizeAttributes,
+                pgSchema: const_1.PG_SCHEMA,
+                pgTableName: modelDef.pgTableName + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp,
+                savingType: "insertOnly",
+                sequelizeAdditionalSettings: Object.assign({ timestamps: false }, (modelDef.modelIndexOptions &&
+                    modelDef.modelIndexOptions.length && {
+                    indexes: modelDef.modelIndexOptions.map((indexDef) => {
+                        return Object.assign(Object.assign({}, indexDef), { name: indexDef.name + SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp });
+                    }),
+                })),
+            }, new golemio_validator_1.JSONSchemaValidator(modelDef.name + "Tmp" + "ModelValidator", modelDef.outputMongooseSchemaObject));
+            return model;
+        });
+        this.logger = Di_2.RopidGtfsContainer.resolve(CoreToken_1.CoreToken.Logger);
+        this.metaModel = Di_2.RopidGtfsContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.RopidGTFSMetadataModel);
+        this.staticFileRedisRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository);
+        this.transformationGtfs = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGTFSTransformation); // TODO is it posible to use factory instead, which already contain this transformation?
+        this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
+        this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
+    }
+    execute(inputData, msgProperties) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const datasetMap = this.ropidGtfsFactory.getDatasetMap();
+            let modelName = (0, HelperTypes_1.datasetFileModelMap)(inputData.dataset, inputData.name);
+            if (!Array.isArray(modelName)) {
+                modelName = [modelName];
+            }
+            let errorMessage;
+            if (inputData.dataset === ropid_gtfs_1.DatasetEnum.PID_GTFS || inputData.dataset === ropid_gtfs_1.DatasetEnum.RUN_NUMBERS) {
+                try {
+                    const sourceStream = this.staticFileRedisRepository.getReadableStream(inputData.filepath);
+                    const transformedDataset = yield this.transformationGtfs.transform({
+                        sourceStream,
+                        name: modelName[0],
+                    });
+                    yield this.saveTransformedDataset(transformedDataset, inputData.dataset, modelName[0]);
+                }
+                catch (err) {
+                    errorMessage = err.message;
+                }
+            }
+            else {
+                try {
+                    const data = yield this.staticFileRedisRepository.get(inputData.filepath);
+                    const transformedDataset = yield datasetMap[inputData.dataset].transformation.transform({
+                        data: datasetMap[inputData.dataset].cacheContentType === HelperTypes_1.CacheContentType.JSON ? JSON.parse(data) : data,
+                        name: inputData.name,
+                    });
+                    if (inputData.dataset === ropid_gtfs_1.DatasetEnum.CIS_STOPS) {
+                        yield this.saveTransformedDataset(transformedDataset.cis_stops, inputData.dataset, modelName[0]);
+                        yield this.saveTransformedDataset(transformedDataset.cis_stop_groups, inputData.dataset, modelName[1]);
+                    }
+                    else {
+                        yield this.saveTransformedDataset(transformedDataset, inputData.dataset, modelName[0]);
+                    }
+                }
+                catch (err) {
+                    errorMessage = err.message;
+                }
+            }
+            if (msgProperties === null || msgProperties === void 0 ? void 0 : msgProperties.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER]) {
+                yield this.gtfsRedisChannel.publishMessage(`${errorMessage !== null && errorMessage !== void 0 ? errorMessage : "OK"} (${modelName})`, {
+                    channelSuffix: msgProperties.headers[HelperTypes_1.ORIGIN_HOSTNAME_HEADER],
+                });
+            }
+            if (errorMessage) {
+                throw new golemio_errors_1.GeneralError(errorMessage, this.constructor.name);
+            }
+        });
+    }
+}
+exports.TransformAndSaveDataTask = TransformAndSaveDataTask;
+//# sourceMappingURL=TransformAndSaveDataTask.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformAndSaveDataTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/TransformAndSaveDataTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA0C;AAC1C,+EAAoE;AACpE,yCAA8F;AAE9F,kFAAqF;AACrF,wCAA2D;AAC3D,kFAAqF;AACrF,0CAA2D;AAC3D,8EAA8D;AAC9D,mEAAuC;AACvC,6EAAiF;AAGjF,wEAAqE;AAGrE,8FAA2F;AAC3F,2EAA6E;AAC7E,6EAAwE;AACxE,mFAAkF;AAElF,qDAAmD;AACnD,uDAM+B;AAE/B,sEAAmE;AAEnE,MAAa,wBAAyB,SAAQ,sBAA4B;IAYtE,YAAY,WAAmB;QAC3B,KAAK,CAAC,WAAW,CAAC,CAAC;QAZP,cAAS,GAAG,sBAAsB,CAAC;QACnC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAC3C,WAAM,GAAG,yCAAmB,CAAC;QA0ErC,2BAAsB,GAAG,CAC7B,eAAoB,EACpB,OAAqB,EACrB,SAA2B,EACd,EAAE;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAEnD,QAAQ,OAAO,EAAE;gBACb,KAAK,wBAAW,CAAC,QAAQ,CAAC;gBAC1B,KAAK,wBAAW,CAAC,WAAW;oBACxB,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,SAA8B,CAAC,CAAC;oBAC5E,MAAM;gBACV;oBACI,MAAM,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAA,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,GAAG,CAAC,CAAC;aAC3E;YAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACrE,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE,kCAAa,CAAC,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;QACtG,CAAC,CAAA,CAAC;QAEM,mBAAc,GAAG,CAAO,IAAY,EAA0B,EAAE;YACpE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,sBAAS,CAAC,IAAuB,CAAC,CAAC,iBAAiB,EAAE;gBACtD,MAAM,KAAK,CAAC,KAAK,CAAC,IAAA,2BAAmB,EAAC,sBAAS,CAAC,IAAuB,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;aAChG;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAS,EAAE,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAA,CAAC;QAEM,oBAAe,GAAG,CAAO,IAAS,EAAE,SAA4B,EAAE,EAAE;YACxE,+BAA+B;YAC/B,MAAM,UAAU,GAAG,iBAAY,CAAC,OAAO,CAAqB,qBAAS,CAAC,iBAAiB,CAAC,CAAC,aAAa,EAAE,CAAC;YACzG,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,iBAAiB,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAQ,CAAC;YAE5F,MAAM,YAAY,GAAG,sBAAS,CAAC,SAA4B,CAAC,CAAC,WAAW,GAAG,6CAAqB,CAAC,GAAG,CAAC;YAErG,+CAA+C;YAC/C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACxC,MAAM,MAAM,GAAG,MAAM;qBAChB,KAAK,CAAC,IAAA,sBAAQ,EAAC,SAAS,iBAAS,KAAK,YAAY,uCAAuC,CAAC,CAAC;qBAC3F,EAAE,CAAC,OAAO,EAAE,CAAO,GAAQ,EAAE,EAAE;oBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,SAAS,MAAM,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBACtF,MAAM,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC7D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAA,CAAC;qBACD,EAAE,CAAC,QAAQ,EAAE,GAAS,EAAE;oBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,GAAG,CAAC,CAAC;oBACpE,MAAM,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC7D,OAAO,OAAO,EAAE,CAAC;gBACrB,CAAC,CAAA,CAAC,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;QAEM,sBAAiB,GAAG,CAAO,IAAY,EAA0B,EAAE;YACvE,MAAM,QAAQ,GAAG,sBAAS,CAAC,IAAuB,CAAC,CAAC;YAEpD,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,yBAAyB,EAAE;gBAC7F,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;aACvC;YAED,MAAM,KAAK,GAAG,IAAI,6BAAa,CAC3B,QAAQ,CAAC,IAAI,GAAG,KAAK,GAAG,OAAO,EAC/B;gBACI,WAAW,EAAE,KAAK;gBAClB,yBAAyB,EAAE,QAAQ,CAAC,yBAAyB;gBAC7D,QAAQ,EAAE,iBAAS;gBACnB,WAAW,EAAE,QAAQ,CAAC,WAAW,GAAG,6CAAqB,CAAC,GAAG;gBAC7D,UAAU,EAAE,YAAY;gBACxB,2BAA2B,kBACvB,UAAU,EAAE,KAAK,IACd,CAAC,QAAQ,CAAC,iBAAiB;oBAC1B,QAAQ,CAAC,iBAAiB,CAAC,MAAM,IAAI;oBACjC,OAAO,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAa,EAAE,EAAE;wBACtD,uCACO,QAAQ,KACX,IAAI,EAAE,QAAQ,CAAC,IAAI,GAAG,6CAAqB,CAAC,GAAG,IACjD;oBACN,CAAC,CAAC;iBACL,CAAC,CACT;aACJ,EACD,IAAI,uCAAmB,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK,GAAG,gBAAgB,EAAE,QAAQ,CAAC,0BAA0B,CAAC,CACzG,CAAC;YAEF,OAAO,KAAK,CAAC;QACjB,CAAC,CAAA,CAAC;QAtJE,IAAI,CAAC,MAAM,GAAG,uBAAkB,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,uBAAkB,CAAC,OAAO,CAAyB,2CAAoB,CAAC,sBAAsB,CAAC,CAAC;QAEjH,IAAI,CAAC,yBAAyB,GAAG,uBAAkB,CAAC,OAAO,CACvD,iDAAuB,CAAC,yBAAyB,CACpD,CAAC;QACF,IAAI,CAAC,kBAAkB,GAAG,uBAAkB,CAAC,OAAO,CAChD,iDAAuB,CAAC,uBAAuB,CAClD,CAAC,CAAC,wFAAwF;QAC3F,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAmB,iDAAuB,CAAC,gBAAgB,CAAC,CAAC;QAC/G,IAAI,CAAC,gBAAgB,GAAG,uBAAkB,CAAC,OAAO,CAAqB,iDAAuB,CAAC,kBAAkB,CAAC,CAAC;IACvH,CAAC;IAEe,OAAO,CAAC,SAAyB,EAAE,aAAiC;;YAChF,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC;YACzD,IAAI,SAAS,GAAG,IAAA,iCAAmB,EAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YACvE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC3B,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;aAC3B;YAED,IAAI,YAAgC,CAAC;YACrC,IAAI,SAAS,CAAC,OAAO,KAAK,wBAAW,CAAC,QAAQ,IAAI,SAAS,CAAC,OAAO,KAAK,wBAAW,CAAC,WAAW,EAAE;gBAC7F,IAAI;oBACA,MAAM,YAAY,GAAG,IAAI,CAAC,yBAAyB,CAAC,iBAAiB,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;oBAC1F,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;wBAC/D,YAAY;wBACZ,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;qBACrB,CAAC,CAAC;oBAEH,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC1F;gBAAC,OAAO,GAAG,EAAE;oBACV,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC;iBAC9B;aACJ;iBAAM;gBACH,IAAI;oBACA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;oBAC1E,MAAM,kBAAkB,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,OAAuB,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC;wBACpG,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,gBAAgB,KAAK,8BAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;wBACxG,IAAI,EAAE,SAAS,CAAC,IAAI;qBACvB,CAAC,CAAC;oBAEH,IAAI,SAAS,CAAC,OAAO,KAAK,wBAAW,CAAC,SAAS,EAAE;wBAC7C,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;wBACjG,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,eAAe,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;qBAC1G;yBAAM;wBACH,MAAM,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;qBAC1F;iBACJ;gBAAC,OAAO,GAAG,EAAE;oBACV,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC;iBAC9B;aACJ;YAED,IAAI,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,OAAO,CAAC,oCAAsB,CAAC,EAAE;gBAChD,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,GAAG,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,IAAI,KAAK,SAAS,GAAG,EAAE;oBACjF,aAAa,EAAE,aAAa,CAAC,OAAO,CAAC,oCAAsB,CAAC;iBAC/D,CAAC,CAAC;aACN;YAED,IAAI,YAAY,EAAE;gBACd,MAAM,IAAI,6BAAY,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAC/D;QACL,CAAC;KAAA;CA0FJ;AArKD,4DAqKC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/HelperTypes.d.ts

@@ -0,0 +1,24 @@
+import { DatasetEnum } from "../../../../RopidGTFSMetadataModel";
+import { DatasetModelName } from "../../../../../../schema-definitions/ropid-gtfs";
+import { IDataSource } from "@golemio/core/dist/integration-engine/datasources/IDataSource";
+import { IFile } from "@golemio/core/dist/integration-engine/datasources/protocol-strategy/IProtocolStrategy";
+import { ITransformation } from "@golemio/core/dist/integration-engine/transformations/ITransformation";
+export declare const ORIGIN_HOSTNAME_HEADER = "x-origin-hostname";
+export type ICachedFileMeta = Omit<IFile, "data">;
+export type ICachedDataset = ICachedFileMeta & {
+    dataset: GTFSDatasets;
+};
+export declare enum CacheContentType {
+    CSV = "CSV",
+    JSON = "JSON"
+}
+export type GTFSDatasets = Exclude<DatasetEnum, DatasetEnum.DEPARTURES_PRESETS>;
+export type DatasetMap = {
+    [key in GTFSDatasets]: {
+        datasource: IDataSource;
+        transformation: ITransformation;
+        cacheContentType: CacheContentType;
+    };
+};
+export declare const isCachedDataset: (obj: any) => obj is ICachedDataset;
+export declare const datasetFileModelMap: (dataset: GTFSDatasets, name: string) => DatasetModelName | DatasetModelName[];

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/HelperTypes.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.datasetFileModelMap = exports.isCachedDataset = exports.CacheContentType = exports.ORIGIN_HOSTNAME_HEADER = void 0;
+const RopidGTFSMetadataModel_1 = require("../../../../RopidGTFSMetadataModel");
+exports.ORIGIN_HOSTNAME_HEADER = "x-origin-hostname";
+var CacheContentType;
+(function (CacheContentType) {
+    CacheContentType["CSV"] = "CSV";
+    CacheContentType["JSON"] = "JSON";
+})(CacheContentType || (exports.CacheContentType = CacheContentType = {}));
+const isCachedDataset = (obj) => {
+    return obj instanceof Object && obj.filepath !== undefined && obj.name !== undefined && obj.dataset !== undefined;
+};
+exports.isCachedDataset = isCachedDataset;
+const datasetFileModelMap = (dataset, name) => {
+    let model;
+    switch (dataset) {
+        case RopidGTFSMetadataModel_1.DatasetEnum.PID_GTFS:
+            model = name;
+            break;
+        case RopidGTFSMetadataModel_1.DatasetEnum.RUN_NUMBERS:
+            model = "run_numbers";
+            break;
+        case RopidGTFSMetadataModel_1.DatasetEnum.OIS_MAPPING:
+            model = "ois";
+            break;
+        case RopidGTFSMetadataModel_1.DatasetEnum.CIS_STOPS:
+            model = ["cis_stops", "cis_stop_groups"];
+            break;
+        default:
+            throw new Error(`Error mapping dataset ${dataset} and source ${name}`);
+    }
+    return model;
+};
+exports.datasetFileModelMap = datasetFileModelMap;
+//# sourceMappingURL=HelperTypes.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/HelperTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HelperTypes.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/HelperTypes.ts"],"names":[],"mappings":";;;AAAA,+EAAoE;AAMvD,QAAA,sBAAsB,GAAG,mBAAmB,CAAC;AAQ1D,IAAY,gBAGX;AAHD,WAAY,gBAAgB;IACxB,+BAAW,CAAA;IACX,iCAAa,CAAA;AACjB,CAAC,EAHW,gBAAgB,gCAAhB,gBAAgB,QAG3B;AAYM,MAAM,eAAe,GAAG,CAAC,GAAQ,EAAyB,EAAE;IAC/D,OAAO,GAAG,YAAY,MAAM,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC;AACtH,CAAC,CAAC;AAFW,QAAA,eAAe,mBAE1B;AAEK,MAAM,mBAAmB,GAAG,CAAC,OAAqB,EAAE,IAAY,EAAyC,EAAE;IAC9G,IAAI,KAAK,CAAC;IACV,QAAQ,OAAO,EAAE;QACb,KAAK,oCAAW,CAAC,QAAQ;YACrB,KAAK,GAAG,IAAyB,CAAC;YAClC,MAAM;QACV,KAAK,oCAAW,CAAC,WAAW;YACxB,KAAK,GAAU,aAAa,CAAC;YAC7B,MAAM;QACV,KAAK,oCAAW,CAAC,WAAW;YACxB,KAAK,GAAU,KAAK,CAAC;YACrB,MAAM;QACV,KAAK,oCAAW,CAAC,SAAS;YACtB,KAAK,GAAG,CAAQ,WAAW,EAAS,iBAAiB,CAAC,CAAC;YACvD,MAAM;QACV;YACI,MAAM,IAAI,KAAK,CAAC,yBAAyB,OAAO,eAAe,IAAI,EAAE,CAAC,CAAC;KAC9E;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AApBW,QAAA,mBAAmB,uBAoB9B"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.d.ts

@@ -0,0 +1,12 @@
+import { SourceTableSuffixEnum } from "../../../../helpers/SourceTableSuffixEnum";
+import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
+export declare class PrecomputedTablesFacade {
+    private readonly logger;
+    /** Precomputed data repositories */
+    private departuresRepository;
+    private minMaxStopSequencesRepository;
+    private tripScheduleRepository;
+    private servicesCalendarRepository;
+    constructor(logger: ILogger);
+    createAndPopulatePrecomputedTmpTables: (sourceTableSuffix: SourceTableSuffixEnum) => Promise<void>;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.js

@@ -0,0 +1,67 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PrecomputedTablesFacade = void 0;
+const DeparturesRepository_1 = require("../../../../data-access/precomputed/DeparturesRepository");
+const MinMaxStopSequencesRepository_1 = require("../../../../data-access/precomputed/MinMaxStopSequencesRepository");
+const ServicesCalendarRepository_1 = require("../../../../data-access/precomputed/ServicesCalendarRepository");
+const TripScheduleRepository_1 = require("../../../../data-access/precomputed/TripScheduleRepository");
+const SourceTableSuffixEnum_1 = require("../../../../helpers/SourceTableSuffixEnum");
+const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
+const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
+let PrecomputedTablesFacade = exports.PrecomputedTablesFacade = class PrecomputedTablesFacade {
+    constructor(logger) {
+        this.logger = logger;
+        this.createAndPopulatePrecomputedTmpTables = (sourceTableSuffix) => __awaiter(this, void 0, void 0, function* () {
+            this.logger.info(`Datasets: creating temp table ${this.servicesCalendarRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.servicesCalendarRepository.createAndPopulate(sourceTableSuffix);
+            this.logger.info(`Datasets: creating indexes on ${this.servicesCalendarRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.servicesCalendarRepository.createIndexes();
+            this.logger.info(`Datasets: creating temp table ${this.minMaxStopSequencesRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.minMaxStopSequencesRepository.createAndPopulate(sourceTableSuffix);
+            this.logger.info(`Datasets: creating indexes on ${this.minMaxStopSequencesRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.minMaxStopSequencesRepository.createIndexes();
+            this.logger.info(`Datasets: creating temp table ${this.tripScheduleRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.tripScheduleRepository.createAndPopulate(sourceTableSuffix);
+            this.logger.info(`Datasets: creating indexes on ${this.tripScheduleRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.tripScheduleRepository.createIndexes();
+            this.logger.info(`Datasets: creating temp table ${this.departuresRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.departuresRepository.createAndPopulate(sourceTableSuffix);
+            this.logger.info(`Datasets: creating indexes on ${this.departuresRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.departuresRepository.createIndexes();
+            this.logger.info(`Datasets: running analyze on  ${this.departuresRepository["tableName"]}${SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp}`);
+            yield this.departuresRepository.analyze();
+        });
+        /** Precomputed data repositories */
+        this.departuresRepository = new DeparturesRepository_1.DeparturesRepository();
+        this.minMaxStopSequencesRepository = new MinMaxStopSequencesRepository_1.MinMaxStopSequencesRepository();
+        this.tripScheduleRepository = new TripScheduleRepository_1.TripScheduleRepository();
+        this.servicesCalendarRepository = new ServicesCalendarRepository_1.ServicesCalendarRepository();
+    }
+};
+exports.PrecomputedTablesFacade = PrecomputedTablesFacade = __decorate([
+    (0, tsyringe_1.injectable)(),
+    __param(0, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
+    __metadata("design:paramtypes", [Object])
+], PrecomputedTablesFacade);
+//# sourceMappingURL=PrecomputedTablesFacade.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PrecomputedTablesFacade.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,mGAAmG;AACnG,qHAAqH;AACrH,+GAA+G;AAC/G,uGAAuG;AACvG,qFAAqF;AACrF,wEAAqE;AAErE,iEAAwE;AAGjE,IAAM,uBAAuB,qCAA7B,MAAM,uBAAuB;IAOhC,YAAsC,MAAgC;QAAf,WAAM,GAAN,MAAM,CAAS;QAQ/D,0CAAqC,GAAG,CAAO,iBAAwC,EAAiB,EAAE;YAC7G,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,iCAAiC,IAAI,CAAC,0BAA0B,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAC9G,CAAC;YACF,MAAM,IAAI,CAAC,0BAA0B,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;YAE3E,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,iCAAiC,IAAI,CAAC,0BAA0B,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAC9G,CAAC;YACF,MAAM,IAAI,CAAC,0BAA0B,CAAC,aAAa,EAAE,CAAC;YAEtD,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,iCAAiC,IAAI,CAAC,6BAA6B,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CACjH,CAAC;YACF,MAAM,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;YAE9E,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,iCAAiC,IAAI,CAAC,6BAA6B,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CACjH,CAAC;YACF,MAAM,IAAI,CAAC,6BAA6B,CAAC,aAAa,EAAE,CAAC;YAEzD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1H,MAAM,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;YAEvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1H,MAAM,IAAI,CAAC,sBAAsB,CAAC,aAAa,EAAE,CAAC;YAElD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAAC,CAAC;YACxH,MAAM,IAAI,CAAC,oBAAoB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAAC,CAAC;YACxH,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,CAAC;YAEhD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,6CAAqB,CAAC,GAAG,EAAE,CAAC,CAAC;YACxH,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,CAAC;QAC9C,CAAC,CAAA,CAAC;QA1CE,oCAAoC;QACpC,IAAI,CAAC,oBAAoB,GAAG,IAAI,2CAAoB,EAAE,CAAC;QACvD,IAAI,CAAC,6BAA6B,GAAG,IAAI,6DAA6B,EAAE,CAAC;QACzE,IAAI,CAAC,sBAAsB,GAAG,IAAI,+CAAsB,EAAE,CAAC;QAC3D,IAAI,CAAC,0BAA0B,GAAG,IAAI,uDAA0B,EAAE,CAAC;IACvE,CAAC;CAsCJ,CAAA;kCAnDY,uBAAuB;IADnC,IAAA,qBAAU,GAAE;IAQI,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;;GAP5B,uBAAuB,CAmDnC"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/RopidGtfsFactory.d.ts

@@ -0,0 +1,13 @@
+import { DatasetMap } from "./HelperTypes";
+export declare class RopidGtfsFactory {
+    private dataSourceGtfs;
+    private dataSourceCisStops;
+    private dataSourceOisMapping;
+    private dataSourceRunNumbers;
+    private transformationGtfs;
+    private transformationCisStops;
+    private transformationOisMapping;
+    private datasetMap;
+    constructor();
+    getDatasetMap(): DatasetMap;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/RopidGtfsFactory.js

@@ -0,0 +1,102 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RopidGtfsFactory = void 0;
+const RopidGTFSMetadataModel_1 = require("../../../../RopidGTFSMetadataModel");
+const RopidGTFSCisStopsTransformation_1 = require("../../../../transformations/RopidGTFSCisStopsTransformation");
+const RopidGTFSOisTransformation_1 = require("../../../../transformations/RopidGTFSOisTransformation");
+const RopidGTFSTransformation_1 = require("../../../../transformations/RopidGTFSTransformation");
+const ropid_gtfs_1 = require("../../../../../../schema-definitions/ropid-gtfs");
+const integration_engine_1 = require("@golemio/core/dist/integration-engine");
+const RetryDataSource_1 = require("@golemio/core/dist/integration-engine/datasources/RetryDataSource");
+const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
+const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
+const HelperTypes_1 = require("./HelperTypes");
+let RopidGtfsFactory = exports.RopidGtfsFactory = class RopidGtfsFactory {
+    constructor() {
+        // Ropid GTFS
+        this.dataSourceGtfs = new RetryDataSource_1.RetryDataSource(ropid_gtfs_1.RopidGTFS.name + "DataSource", new integration_engine_1.FTPProtocolStrategy({
+            filename: integration_engine_1.config.datasources.RopidGTFSFilename,
+            targetType: integration_engine_1.FTPTargetType.COMPRESSED,
+            returnType: integration_engine_1.FTPReturnType.Stream,
+            path: integration_engine_1.config.datasources.RopidGTFSPath,
+            url: integration_engine_1.config.datasources.RopidFTP,
+            whitelistedFiles: [
+                "agency.txt",
+                "calendar.txt",
+                "calendar_dates.txt",
+                "shapes.txt",
+                "stop_times.txt",
+                "stops.txt",
+                "routes.txt",
+                "trips.txt",
+            ],
+            encoding: "utf8",
+        }), new integration_engine_1.JSONDataTypeStrategy({ resultsPath: "" }), null);
+        this.transformationGtfs = new RopidGTFSTransformation_1.RopidGTFSTransformation();
+        // Ropid GTFS Run Numbers Datasource
+        this.dataSourceRunNumbers = new RetryDataSource_1.RetryDataSource(ropid_gtfs_1.RopidGTFS.run_numbers.name + "DataSource", new integration_engine_1.FTPProtocolStrategy({
+            filename: integration_engine_1.config.datasources.RopidGTFSRunNumbersFilename,
+            returnType: integration_engine_1.FTPReturnType.Stream,
+            path: integration_engine_1.config.datasources.RopidGTFSRunNumbersPath,
+            url: integration_engine_1.config.datasources.RopidFTP,
+            encoding: "utf8",
+        }), new integration_engine_1.JSONDataTypeStrategy({ resultsPath: "" }), null);
+        // Ropid GTFS CIS STOPS Datasource
+        const cisStopsTypeStrategy = new integration_engine_1.JSONDataTypeStrategy({ resultsPath: "stopGroups" });
+        cisStopsTypeStrategy.setFilter((item) => item.cis !== 0);
+        this.dataSourceCisStops = new RetryDataSource_1.RetryDataSource(ropid_gtfs_1.RopidGTFS.name + "CisStops", new integration_engine_1.FTPProtocolStrategy({
+            filename: integration_engine_1.config.datasources.RopidGTFSCisStopsFilename,
+            path: integration_engine_1.config.datasources.RopidGTFSCisStopsPath,
+            url: integration_engine_1.config.datasources.RopidFTP,
+            encoding: "utf8",
+        }), cisStopsTypeStrategy, null);
+        this.transformationCisStops = new RopidGTFSCisStopsTransformation_1.RopidGTFSCisStopsTransformation();
+        // Ropid GTFS Ois Datasource
+        this.dataSourceOisMapping = new RetryDataSource_1.RetryDataSource(ropid_gtfs_1.RopidGTFS.name + "Ois", new integration_engine_1.FTPProtocolStrategy({
+            filename: integration_engine_1.config.datasources.RopidGTFSOisFilename,
+            path: integration_engine_1.config.datasources.RopidGTFSOisPath,
+            url: integration_engine_1.config.datasources.RopidFTP,
+            encoding: "utf8",
+        }), new integration_engine_1.JSONDataTypeStrategy({ resultsPath: "" }), new golemio_validator_1.JSONSchemaValidator(ropid_gtfs_1.RopidGTFS.ois.name + "DataSource", ropid_gtfs_1.RopidGTFS.ois.datasourceJsonSchema));
+        this.transformationOisMapping = new RopidGTFSOisTransformation_1.RopidGTFSOisMappingTransformation();
+        this.datasetMap = {
+            [RopidGTFSMetadataModel_1.DatasetEnum.PID_GTFS]: {
+                datasource: this.dataSourceGtfs,
+                transformation: this.transformationGtfs,
+                cacheContentType: HelperTypes_1.CacheContentType.CSV,
+            },
+            [RopidGTFSMetadataModel_1.DatasetEnum.RUN_NUMBERS]: {
+                datasource: this.dataSourceRunNumbers,
+                transformation: this.transformationGtfs,
+                cacheContentType: HelperTypes_1.CacheContentType.CSV,
+            },
+            [RopidGTFSMetadataModel_1.DatasetEnum.CIS_STOPS]: {
+                datasource: this.dataSourceCisStops,
+                transformation: this.transformationCisStops,
+                cacheContentType: HelperTypes_1.CacheContentType.JSON,
+            },
+            [RopidGTFSMetadataModel_1.DatasetEnum.OIS_MAPPING]: {
+                datasource: this.dataSourceOisMapping,
+                transformation: this.transformationOisMapping,
+                cacheContentType: HelperTypes_1.CacheContentType.JSON,
+            },
+        };
+    }
+    getDatasetMap() {
+        return this.datasetMap;
+    }
+};
+exports.RopidGtfsFactory = RopidGtfsFactory = __decorate([
+    (0, tsyringe_1.injectable)(),
+    __metadata("design:paramtypes", [])
+], RopidGtfsFactory);
+//# sourceMappingURL=RopidGtfsFactory.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/RopidGtfsFactory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RopidGtfsFactory.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/helpers/RopidGtfsFactory.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+EAAoE;AACpE,iHAAiH;AACjH,uGAA8G;AAC9G,iGAAiG;AACjG,gFAA4C;AAC5C,8EAO+C;AAC/C,uGAAoG;AACpG,mFAAkF;AAClF,iEAAgE;AAChE,+CAA6D;AAGtD,IAAM,gBAAgB,8BAAtB,MAAM,gBAAgB;IAYzB;QACI,aAAa;QACb,IAAI,CAAC,cAAc,GAAG,IAAI,iCAAe,CACrC,sBAAS,CAAC,IAAI,GAAG,YAAY,EAC7B,IAAI,wCAAmB,CAAC;YACpB,QAAQ,EAAE,2BAAM,CAAC,WAAW,CAAC,iBAAiB;YAC9C,UAAU,EAAE,kCAAa,CAAC,UAAU;YACpC,UAAU,EAAE,kCAAa,CAAC,MAAM;YAChC,IAAI,EAAE,2BAAM,CAAC,WAAW,CAAC,aAAa;YACtC,GAAG,EAAE,2BAAM,CAAC,WAAW,CAAC,QAAQ;YAChC,gBAAgB,EAAE;gBACd,YAAY;gBACZ,cAAc;gBACd,oBAAoB;gBACpB,YAAY;gBACZ,gBAAgB;gBAChB,WAAW;gBACX,YAAY;gBACZ,WAAW;aACd;YACD,QAAQ,EAAE,MAAM;SACnB,CAAC,EACF,IAAI,yCAAoB,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,EAC7C,IAAW,CACd,CAAC;QACF,IAAI,CAAC,kBAAkB,GAAG,IAAI,iDAAuB,EAAE,CAAC;QAExD,oCAAoC;QACpC,IAAI,CAAC,oBAAoB,GAAG,IAAI,iCAAe,CAC3C,sBAAS,CAAC,WAAW,CAAC,IAAI,GAAG,YAAY,EACzC,IAAI,wCAAmB,CAAC;YACpB,QAAQ,EAAE,2BAAM,CAAC,WAAW,CAAC,2BAA2B;YACxD,UAAU,EAAE,kCAAa,CAAC,MAAM;YAChC,IAAI,EAAE,2BAAM,CAAC,WAAW,CAAC,uBAAuB;YAChD,GAAG,EAAE,2BAAM,CAAC,WAAW,CAAC,QAAQ;YAChC,QAAQ,EAAE,MAAM;SACnB,CAAC,EACF,IAAI,yCAAoB,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,EAC7C,IAAW,CACd,CAAC;QAEF,kCAAkC;QAClC,MAAM,oBAAoB,GAAG,IAAI,yCAAoB,CAAC,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;QACrF,oBAAoB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,kBAAkB,GAAG,IAAI,iCAAe,CACzC,sBAAS,CAAC,IAAI,GAAG,UAAU,EAC3B,IAAI,wCAAmB,CAAC;YACpB,QAAQ,EAAE,2BAAM,CAAC,WAAW,CAAC,yBAAyB;YACtD,IAAI,EAAE,2BAAM,CAAC,WAAW,CAAC,qBAAqB;YAC9C,GAAG,EAAE,2BAAM,CAAC,WAAW,CAAC,QAAQ;YAChC,QAAQ,EAAE,MAAM;SACnB,CAAC,EACF,oBAAoB,EACpB,IAAW,CACd,CAAC;QACF,IAAI,CAAC,sBAAsB,GAAG,IAAI,iEAA+B,EAAE,CAAC;QAEpE,4BAA4B;QAC5B,IAAI,CAAC,oBAAoB,GAAG,IAAI,iCAAe,CAC3C,sBAAS,CAAC,IAAI,GAAG,KAAK,EACtB,IAAI,wCAAmB,CAAC;YACpB,QAAQ,EAAE,2BAAM,CAAC,WAAW,CAAC,oBAAoB;YACjD,IAAI,EAAE,2BAAM,CAAC,WAAW,CAAC,gBAAgB;YACzC,GAAG,EAAE,2BAAM,CAAC,WAAW,CAAC,QAAQ;YAChC,QAAQ,EAAE,MAAM;SACnB,CAAC,EACF,IAAI,yCAAoB,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,EAC7C,IAAI,uCAAmB,CAAC,sBAAS,CAAC,GAAG,CAAC,IAAI,GAAG,YAAY,EAAE,sBAAS,CAAC,GAAG,CAAC,oBAAoB,CAAC,CACjG,CAAC;QACF,IAAI,CAAC,wBAAwB,GAAG,IAAI,8DAAiC,EAAE,CAAC;QAExE,IAAI,CAAC,UAAU,GAAG;YACd,CAAC,oCAAW,CAAC,QAAQ,CAAC,EAAE;gBACpB,UAAU,EAAE,IAAI,CAAC,cAAc;gBAC/B,cAAc,EAAE,IAAI,CAAC,kBAAkB;gBACvC,gBAAgB,EAAE,8BAAgB,CAAC,GAAG;aACzC;YACD,CAAC,oCAAW,CAAC,WAAW,CAAC,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,oBAAoB;gBACrC,cAAc,EAAE,IAAI,CAAC,kBAAkB;gBACvC,gBAAgB,EAAE,8BAAgB,CAAC,GAAG;aACzC;YACD,CAAC,oCAAW,CAAC,SAAS,CAAC,EAAE;gBACrB,UAAU,EAAE,IAAI,CAAC,kBAAkB;gBACnC,cAAc,EAAE,IAAI,CAAC,sBAAsB;gBAC3C,gBAAgB,EAAE,8BAAgB,CAAC,IAAI;aAC1C;YACD,CAAC,oCAAW,CAAC,WAAW,CAAC,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,oBAAoB;gBACrC,cAAc,EAAE,IAAI,CAAC,wBAAwB;gBAC7C,gBAAgB,EAAE,8BAAgB,CAAC,IAAI;aAC1C;SACJ,CAAC;IACN,CAAC;IAEM,aAAa;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;CACJ,CAAA;2BA9GY,gBAAgB;IAD5B,IAAA,qBAAU,GAAE;;GACA,gBAAgB,CA8G5B"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/interfaces/ICheckForNewDataInput.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ICheckForNewDataInput.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/interfaces/ICheckForNewDataInput.ts"],"names":[],"mappings":""}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/interfaces/IDatasetsInput.d.ts

@@ -0,0 +1,4 @@
+import { GTFSDatasets } from "../helpers/HelperTypes";
+export interface IDatasetsInput {
+    datasets: GTFSDatasets[];
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/interfaces/IDatasetsInput.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=IDatasetsInput.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/interfaces/IDatasetsInput.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IDatasetsInput.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/interfaces/IDatasetsInput.ts"],"names":[],"mappings":""}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/schema/CachedDatasetSchema.d.ts

@@ -0,0 +1,8 @@
+import { GTFSDatasets, ICachedDataset } from "../helpers/HelperTypes";
+export declare class CachedDatasetSchema implements ICachedDataset {
+    filepath: string;
+    name: string;
+    mtime?: string | undefined;
+    type?: string | undefined;
+    dataset: GTFSDatasets;
+}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/schema/CachedDatasetSchema.js

@@ -0,0 +1,40 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CachedDatasetSchema = void 0;
+const ropid_gtfs_1 = require("../../../..");
+const class_validator_1 = require("@golemio/core/dist/shared/class-validator");
+class CachedDatasetSchema {
+}
+exports.CachedDatasetSchema = CachedDatasetSchema;
+__decorate([
+    (0, class_validator_1.IsString)(),
+    __metadata("design:type", String)
+], CachedDatasetSchema.prototype, "filepath", void 0);
+__decorate([
+    (0, class_validator_1.IsString)(),
+    __metadata("design:type", String)
+], CachedDatasetSchema.prototype, "name", void 0);
+__decorate([
+    (0, class_validator_1.IsOptional)(),
+    (0, class_validator_1.IsString)(),
+    __metadata("design:type", Object)
+], CachedDatasetSchema.prototype, "mtime", void 0);
+__decorate([
+    (0, class_validator_1.IsOptional)(),
+    (0, class_validator_1.IsString)(),
+    __metadata("design:type", Object)
+], CachedDatasetSchema.prototype, "type", void 0);
+__decorate([
+    (0, class_validator_1.IsEnum)(ropid_gtfs_1.DatasetEnum),
+    __metadata("design:type", String)
+], CachedDatasetSchema.prototype, "dataset", void 0);
+//# sourceMappingURL=CachedDatasetSchema.js.map

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/schema/CachedDatasetSchema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CachedDatasetSchema.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/ropid-gtfs/workers/timetables/tasks/schema/CachedDatasetSchema.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,4CAA6C;AAC7C,+EAAyF;AAGzF,MAAa,mBAAmB;CAa/B;AAbD,kDAaC;AAXG;IADC,IAAA,0BAAQ,GAAE;;qDACO;AAElB;IADC,IAAA,0BAAQ,GAAE;;iDACG;AAGd;IAFC,IAAA,4BAAU,GAAE;IACZ,IAAA,0BAAQ,GAAE;;kDACgB;AAG3B;IAFC,IAAA,4BAAU,GAAE;IACZ,IAAA,0BAAQ,GAAE;;iDACe;AAE1B;IADC,IAAA,wBAAM,EAAC,wBAAW,CAAC;;oDACG"}

package/dist/integration-engine/ropid-gtfs/workers/timetables/tasks/schema/CheckForNewDataMessageValidation.d.ts

@@ -0,0 +1,4 @@
+import { ICheckForNewDataInput } from "../interfaces/ICheckForNewDataInput";
+export declare class CheckForNewDataMessageValidation implements ICheckForNewDataInput {
+    forceRefresh?: boolean | undefined;
+}