@golemio/pid 2.8.1-dev.943156853 → 2.8.1

package/db/example/12_metro_line_A_gtfs_trips.sql

@@ -11,10 +11,6 @@ INSERT INTO ropidgtfs_stops_actual (level_id,location_type,parent_station,platfo
 	 ('U462L3',0,'U462S1','M2',NULL,NULL,'U462Z102P',50.09583,14.34836,'Nádraží Veleslavín',NULL,NULL,1,'P',462,102,-1,'2022-10-24 10:21:10.47+02',NULL,-1,'2022-10-24 10:21:10.47+02',NULL)
         ON CONFLICT DO NOTHING;
 
-INSERT INTO ropidgtfs_stops_actual (stop_id,level_id,location_type,parent_station,platform_code,stop_code,stop_desc,stop_lat,stop_lon,stop_name,stop_timezone,stop_url,wheelchair_boarding,zone_id,asw_node_id,asw_stop_id,create_batch_id,created_at,created_by,update_batch_id,updated_at,updated_by) VALUES
-	 ('U476Z101P','U476L2',0,'U476S1','1',NULL,NULL,50.07531,14.43976,'Náměstí Míru',NULL,NULL,2,'P',476,101,NULL,'2023-07-20 05:38:03.527',NULL,NULL,'2023-07-20 05:38:03.527',NULL),
-	 ('U209Z101P','U209L2',0,'U209S1','1',NULL,NULL,50.07769,14.45077,'Jiřího z Poděbrad',NULL,NULL,0,'P',209,101,NULL,'2023-07-20 05:38:03.527',NULL,NULL,'2023-07-20 05:38:03.527',NULL);
-
 INSERT INTO ropidgtfs_stop_times_actual (arrival_time,arrival_time_seconds,departure_time,departure_time_seconds,drop_off_type,pickup_type,shape_dist_traveled,stop_headsign,stop_id,stop_sequence,timepoint,trip_id,create_batch_id,created_at,created_by,update_batch_id,updated_at,updated_by) VALUES
 	 ('9:55:10',NULL,'9:55:10',NULL,'0','0',0.0,NULL,'U1071Z101P',1,NULL,'991_77_220901',-1,'2022-10-24 10:21:13.104+02',NULL,-1,'2022-10-24 10:21:13.104+02',NULL),
 	 ('9:57:10',NULL,'9:57:40',NULL,'0','0',1.142977,NULL,'U953Z102P',2,NULL,'991_77_220901',-1,'2022-10-24 10:21:13.104+02',NULL,-1,'2022-10-24 10:21:13.104+02',NULL),

package/dist/integration-engine/index.js

@@ -31,7 +31,6 @@ const PresetWorker_1 = require("./ropid-gtfs/workers/presets/PresetWorker");
 const RailtrackWorker_1 = require("./ropid-gtfs/workers/railtrack/RailtrackWorker");
 const RopidVYMIWorker_1 = require("./ropid-vymi/workers/RopidVYMIWorker");
 const vehicle_positions_1 = require("./vehicle-positions");
-const TimetableWorker_1 = require("./ropid-gtfs/workers/timetables/TimetableWorker");
 __exportStar(require("./queueDefinitions"), exports);
 exports.RopidGTFS = __importStar(require("./ropid-gtfs"));
 exports.RopidVYMI = __importStar(require("./ropid-vymi"));
@@ -46,7 +45,6 @@ exports.workers = [
     //#region ROPID GTFS
     RailtrackWorker_1.RailtrackWorker,
     PresetWorker_1.PresetWorker,
-    TimetableWorker_1.TimetableWorker,
     //#endregion
     //#region ROPID VYMI
     RopidVYMIWorker_1.RopidVYMIWorker,

package/dist/integration-engine/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/integration-engine/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,4EAAyE;AACzE,oFAAiF;AACjF,0EAAuE;AACvE,2DAAiG;AACjG,qFAAkF;AAElF,qDAAmC;AACnC,0DAA0C;AAC1C,0DAA0C;AAC1C,wEAAwD;AAE3C,QAAA,OAAO,GAAoC;IACpD,2BAA2B;IAC3B,4BAAQ;IACR,oCAAgB;IAChB,8BAAU;IACV,sCAAkB;IAClB,YAAY;IAEZ,oBAAoB;IACpB,iCAAe;IACf,2BAAY;IACZ,iCAAe;IACf,YAAY;IAEZ,oBAAoB;IACpB,iCAAe;IACf,YAAY;CACf,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/integration-engine/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,4EAAyE;AACzE,oFAAiF;AACjF,0EAAuE;AACvE,2DAAiG;AAEjG,qDAAmC;AACnC,0DAA0C;AAC1C,0DAA0C;AAC1C,wEAAwD;AAE3C,QAAA,OAAO,GAAoC;IACpD,2BAA2B;IAC3B,4BAAQ;IACR,oCAAgB;IAChB,8BAAU;IACV,sCAAkB;IAClB,YAAY;IAEZ,oBAAoB;IACpB,iCAAe;IACf,2BAAY;IACZ,YAAY;IAEZ,oBAAoB;IACpB,iCAAe;IACf,YAAY;CACf,CAAC"}

package/dist/integration-engine/queueDefinitions.js

@@ -13,6 +13,56 @@ const queueDefinitions = [
         name: ropid_gtfs_1.RopidGTFS.name,
         queuePrefix: config_1.config.RABBIT_EXCHANGE_NAME + "." + ropid_gtfs_1.RopidGTFS.name.toLowerCase(),
         queues: [
+            {
+                name: "checkForNewData",
+                options: {
+                    deadLetterExchange: config_1.config.RABBIT_EXCHANGE_NAME,
+                    deadLetterRoutingKey: "dead",
+                    messageTtl: 19 * 60 * 1000, // 19 minutes
+                },
+                worker: LegacyRopidGTFSWorker_1.LegacyRopidGTFSWorker,
+                workerMethod: "checkForNewData",
+            },
+            {
+                name: "downloadDatasets",
+                options: {
+                    deadLetterExchange: config_1.config.RABBIT_EXCHANGE_NAME,
+                    deadLetterRoutingKey: "dead",
+                    messageTtl: 59 * 60 * 1000, // 59 minutes
+                },
+                worker: LegacyRopidGTFSWorker_1.LegacyRopidGTFSWorker,
+                workerMethod: "downloadDatasets",
+            },
+            {
+                name: "transformAndSaveData",
+                options: {
+                    deadLetterExchange: config_1.config.RABBIT_EXCHANGE_NAME,
+                    deadLetterRoutingKey: "dead",
+                    messageTtl: 23 * 60 * 60 * 1000, // 23 hours
+                },
+                worker: LegacyRopidGTFSWorker_1.LegacyRopidGTFSWorker,
+                workerMethod: "transformAndSaveData",
+            },
+            {
+                name: "checkSavedRowsAndReplaceTables",
+                options: {
+                    deadLetterExchange: config_1.config.RABBIT_EXCHANGE_NAME,
+                    deadLetterRoutingKey: "dead",
+                    messageTtl: 23 * 60 * 60 * 1000, // 23 hours
+                },
+                worker: LegacyRopidGTFSWorker_1.LegacyRopidGTFSWorker,
+                workerMethod: "checkSavedRowsAndReplaceTables",
+            },
+            {
+                name: "refreshPrecomputedTables",
+                options: {
+                    deadLetterExchange: config_1.config.RABBIT_EXCHANGE_NAME,
+                    deadLetterRoutingKey: "dead",
+                    messageTtl: 59 * 60 * 1000, // 59 minutes
+                },
+                worker: LegacyRopidGTFSWorker_1.LegacyRopidGTFSWorker,
+                workerMethod: "refreshPrecomputedTables",
+            },
             {
                 name: "checkForNewDeparturesPresets",
                 options: {

package/dist/integration-engine/queueDefinitions.js.map

@@ -1 +1 @@
-{"version":3,"file":"queueDefinitions.js","sourceRoot":"","sources":["../../src/integration-engine/queueDefinitions.ts"],"names":[],"mappings":";;;AAAA,8EAA6E;AAC7E,iEAA4C;AAC5C,yEAAsE;AAGtE;;GAEG;AACH,MAAM,gBAAgB,GAAuB;IACzC,+BAA+B;IAC/B;QACI,IAAI,EAAE,sBAAS,CAAC,IAAI;QACpB,WAAW,EAAE,eAAM,CAAC,oBAAoB,GAAG,GAAG,GAAG,sBAAS,CAAC,IAAI,CAAC,WAAW,EAAE;QAC7E,MAAM,EAAE;YACJ;gBACI,IAAI,EAAE,8BAA8B;gBACpC,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;iBAC1C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,8BAA8B;aAC/C;YACD;gBACI,IAAI,EAAE,2BAA2B;gBACjC,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;iBAC1C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,2BAA2B;aAC5C;SACJ;KACJ;CACJ,CAAC;AAEO,4CAAgB"}
+{"version":3,"file":"queueDefinitions.js","sourceRoot":"","sources":["../../src/integration-engine/queueDefinitions.ts"],"names":[],"mappings":";;;AAAA,8EAA6E;AAC7E,iEAA4C;AAC5C,yEAAsE;AAGtE;;GAEG;AACH,MAAM,gBAAgB,GAAuB;IACzC,+BAA+B;IAC/B;QACI,IAAI,EAAE,sBAAS,CAAC,IAAI;QACpB,WAAW,EAAE,eAAM,CAAC,oBAAoB,GAAG,GAAG,GAAG,sBAAS,CAAC,IAAI,CAAC,WAAW,EAAE;QAC7E,MAAM,EAAE;YACJ;gBACI,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,aAAa;iBAC5C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,iBAAiB;aAClC;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,aAAa;iBAC5C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,kBAAkB;aACnC;YACD;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,WAAW;iBAC/C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,sBAAsB;aACvC;YACD;gBACI,IAAI,EAAE,gCAAgC;gBACtC,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,WAAW;iBAC/C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,gCAAgC;aACjD;YACD;gBACI,IAAI,EAAE,0BAA0B;gBAChC,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,aAAa;iBAC5C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,0BAA0B;aAC3C;YACD;gBACI,IAAI,EAAE,8BAA8B;gBACpC,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;iBAC1C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,8BAA8B;aAC/C;YACD;gBACI,IAAI,EAAE,2BAA2B;gBACjC,OAAO,EAAE;oBACL,kBAAkB,EAAE,eAAM,CAAC,oBAAoB;oBAC/C,oBAAoB,EAAE,MAAM;oBAC5B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;iBAC1C;gBACD,MAAM,EAAE,6CAAqB;gBAC7B,YAAY,EAAE,2BAA2B;aAC5C;SACJ;KACJ;CACJ,CAAC;AAEO,4CAAgB"}

package/dist/integration-engine/ropid-gtfs/LegacyRopidGTFSWorker.d.ts

@@ -1,19 +1,82 @@
 import { IRopidDeparturesPreset } from "../../schema-definitions/ropid-departures-presets";
+import { DatasetModelName } from "../../schema-definitions/ropid-gtfs";
 import { BaseWorker } from "@golemio/core/dist/integration-engine/workers";
+import { Message as AMQPMessage } from "amqplib";
+import { DatasetEnum } from ".";
+type GTFSDatasets = Exclude<DatasetEnum, DatasetEnum.DEPARTURES_PRESETS>;
 export interface IDeparturesPresetsData {
     data: IRopidDeparturesPreset[];
 }
+export declare const datasetFileModelMap: (dataset: GTFSDatasets, name: string) => DatasetModelName | DatasetModelName[];
 export declare class LegacyRopidGTFSWorker extends BaseWorker {
+    private static ORIGIN_HOSTNAME_HEADER;
+    private dataSourceGtfs;
+    private dataSourceCisStops;
+    private dataSourceOisMapping;
+    private dataSourceRunNumbers;
     private dataSourceDeparturesPresets;
+    private transformationGtfs;
+    private transformationCisStops;
+    private transformationOisMapping;
+    /** Precomputed data repositories */
+    private departuresRepository;
+    private minMaxStopSequencesRepository;
+    private tripScheduleRepository;
+    private servicesCalendarRepository;
     private metaModel;
+    private staticFileRedisRepository;
+    private dataCacheManager;
+    private gtfsRedisChannel;
+    private datasetMap;
     private readonly queuePrefix;
     constructor();
+    /**
+     * Queue listener method - triggers entire flow
+     */
+    checkForNewData: (msg: AMQPMessage) => Promise<void>;
+    /**
+     * Queue listener method - triggers dataset download
+     * @param {object} msg
+     * @param {Buffer} msg.content - array of dataset names, empty to force downloading all
+     */
+    downloadDatasets: (msg: AMQPMessage) => Promise<void>;
+    /**
+     * Helper method - download and persist dataset
+     */
+    private downloadAndPersistDatasetFiles;
     /**
      * Queue listener method - triggers for new presets only
      */
     checkForNewDeparturesPresets: () => Promise<void>;
+    /**
+     * Queue listener method - transform and save dataset item to tmp
+     */
+    transformAndSaveData: (msg: AMQPMessage) => Promise<void>;
+    private saveTransformedDataset;
+    /**
+     * Queue listener method - verify tmp tables and replace public
+     */
+    checkSavedRowsAndReplaceTables: (msg: AMQPMessage) => Promise<boolean>;
+    refreshPrecomputedTables: () => Promise<void>;
     /**
      * Queue listener method - triggers processing of DEPARTURES_PRESETS dataset
      */
     downloadDeparturesPresets: () => Promise<void>;
+    /**
+     * Validate dataset list against dataset enum
+     *
+     * @param {string[]} names - names of datasets
+     */
+    private validateDatasetNames;
+    /**
+     * Validate name against dataset enum
+     *
+     * @param {string} name - name of datasets
+     */
+    private validateDatasetName;
+    private streamDataToTmp;
+    private getTmpModelByName;
+    private createTmpModel;
+    private createAndPopulatePrecomputedTmpTables;
 }
+export {};