@golemio/public-transport 0.1.0-dev.2980129203 → 0.1.0-dev.2980595950
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.d.ts +5 -0
- package/dist/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.js +18 -0
- package/dist/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.js.map +1 -1
- package/dist/integration-engine/gtfs/ioc/Di.js +2 -0
- package/dist/integration-engine/gtfs/ioc/Di.js.map +1 -1
- package/dist/integration-engine/gtfs/ioc/GtfsContainerToken.d.ts +1 -0
- package/dist/integration-engine/gtfs/ioc/GtfsContainerToken.js +1 -0
- package/dist/integration-engine/gtfs/ioc/GtfsContainerToken.js.map +1 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.d.ts +3 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js +19 -12
- package/dist/integration-engine/gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js.map +1 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/DownloadDatasetsTask.js +1 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/DownloadDatasetsTask.js.map +1 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.d.ts +23 -5
- package/dist/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js +81 -16
- package/dist/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js.map +1 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.d.ts +4 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.js +6 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.js.map +1 -1
- package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.d.ts +16 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.js +68 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.js.map +1 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/interfaces/IRefreshPrecomputedTablesInput.d.ts +5 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/interfaces/IRefreshPrecomputedTablesInput.js +3 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/interfaces/IRefreshPrecomputedTablesInput.js.map +1 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/schema/RefreshPrecomputedTablesSchema.d.ts +5 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/schema/RefreshPrecomputedTablesSchema.js +33 -0
- package/dist/integration-engine/gtfs/workers/timetables/tasks/schema/RefreshPrecomputedTablesSchema.js.map +1 -0
- package/docs/implementation_documentation.md +13 -3
- package/package.json +1 -1
package/dist/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.d.ts
CHANGED
|
@@ -2,5 +2,10 @@ import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine/mod
|
|
|
2
2
|
import { SourceTableSuffixEnum } from "../../helpers/SourceTableSuffixEnum";
|
|
3
3
|
export declare class ServicesCalendarRepository extends PostgresModel implements IModel {
|
|
4
4
|
constructor();
|
|
5
|
+
/**
|
|
6
|
+
* Tells whether the feed's live services calendar already covers today's window, i.e. it was built
|
|
7
|
+
* today (the window starts yesterday, see populate). Uses the same date expression as populate.
|
|
8
|
+
*/
|
|
9
|
+
hasCurrentWindow(feedCode: string): Promise<boolean>;
|
|
5
10
|
populate: (sourceTableSuffix: SourceTableSuffixEnum, feedCode: string, feedId: number) => Promise<void>;
|
|
6
11
|
}
|
|
@@ -6,6 +6,7 @@ const gtfs_1 = require("../../../../schema-definitions/gtfs");
|
|
|
6
6
|
const helpers_1 = require("@golemio/core/dist/integration-engine/helpers");
|
|
7
7
|
const models_1 = require("@golemio/core/dist/integration-engine/models");
|
|
8
8
|
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
9
|
+
const sequelize_1 = require("@golemio/core/dist/shared/sequelize");
|
|
9
10
|
const FeedTableSuffix_1 = require("../../helpers/FeedTableSuffix");
|
|
10
11
|
const SourceTableSuffixEnum_1 = require("../../helpers/SourceTableSuffixEnum");
|
|
11
12
|
class ServicesCalendarRepository extends models_1.PostgresModel {
|
|
@@ -95,6 +96,23 @@ class ServicesCalendarRepository extends models_1.PostgresModel {
|
|
|
95
96
|
}
|
|
96
97
|
};
|
|
97
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Tells whether the feed's live services calendar already covers today's window, i.e. it was built
|
|
101
|
+
* today (the window starts yesterday, see populate). Uses the same date expression as populate.
|
|
102
|
+
*/
|
|
103
|
+
async hasCurrentWindow(feedCode) {
|
|
104
|
+
const connection = this.sequelizeModel.sequelize;
|
|
105
|
+
const actualTable = gtfs_1.Gtfs.servicesCalendar.pgTableName + (0, FeedTableSuffix_1.feedTableSuffix)(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Actual, feedCode);
|
|
106
|
+
const [tableExists] = await connection.query(`SELECT to_regclass('${const_1.PG_SCHEMA}."${actualTable}"') IS NOT NULL AS table_exists;`, { type: sequelize_1.QueryTypes.SELECT });
|
|
107
|
+
if (!tableExists.table_exists) {
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
const [windowStart] = await connection.query(`SELECT EXISTS (
|
|
111
|
+
SELECT 1 FROM "${const_1.PG_SCHEMA}"."${actualTable}"
|
|
112
|
+
WHERE date = (CURRENT_TIMESTAMP - interval '1 day')::date
|
|
113
|
+
) AS is_current;`, { type: sequelize_1.QueryTypes.SELECT });
|
|
114
|
+
return windowStart.is_current;
|
|
115
|
+
}
|
|
98
116
|
}
|
|
99
117
|
exports.ServicesCalendarRepository = ServicesCalendarRepository;
|
|
100
118
|
//# sourceMappingURL=ServicesCalendarRepository.js.map
|
package/dist/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ServicesCalendarRepository.js","sourceRoot":"","sources":["../../../../../src/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.ts"],"names":[],"mappings":";;;AAAA,gEAAuC;AACvC,8DAAiC;AACjC,2EAAoE;AACpE,yEAAqF;AACrF,mFAAkF;AAClF,mEAAgE;AAChE,+EAA4E;AAE5E,MAAa,0BAA2B,SAAQ,sBAAa;IACzD;QACI,KAAK,CACD,4BAA4B,EAC5B;YACI,yBAAyB,EAAE,WAAI,CAAC,gBAAgB,CAAC,yBAAyB;YAC1E,WAAW,EAAE,WAAI,CAAC,gBAAgB,CAAC,WAAW;YAC9C,QAAQ,EAAE,iBAAS;YACnB,UAAU,EAAE,YAAY;YACxB,kBAAkB,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC;SACzD,EACD,IAAI,uCAAmB,CAAC,qCAAqC,EAAE,WAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CACzG,CAAC;
|
|
1
|
+
{"version":3,"file":"ServicesCalendarRepository.js","sourceRoot":"","sources":["../../../../../src/integration-engine/gtfs/data-access/precomputed/ServicesCalendarRepository.ts"],"names":[],"mappings":";;;AAAA,gEAAuC;AACvC,8DAAiC;AACjC,2EAAoE;AACpE,yEAAqF;AACrF,mFAAkF;AAClF,mEAAiE;AACjE,mEAAgE;AAChE,+EAA4E;AAE5E,MAAa,0BAA2B,SAAQ,sBAAa;IACzD;QACI,KAAK,CACD,4BAA4B,EAC5B;YACI,yBAAyB,EAAE,WAAI,CAAC,gBAAgB,CAAC,yBAAyB;YAC1E,WAAW,EAAE,WAAI,CAAC,gBAAgB,CAAC,WAAW;YAC9C,QAAQ,EAAE,iBAAS;YACnB,UAAU,EAAE,YAAY;YACxB,kBAAkB,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC;SACzD,EACD,IAAI,uCAAmB,CAAC,qCAAqC,EAAE,WAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CACzG,CAAC;QA6BC,aAAQ,GAAG,KAAK,EAAE,iBAAwC,EAAE,QAAgB,EAAE,MAAc,EAAiB,EAAE;YAClH,MAAM,YAAY,GAAG,IAAA,iCAAe,EAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;YAClE,6FAA6F;YAC7F,iGAAiG;YACjG,6FAA6F;YAC7F,oGAAoG;YACpG,MAAM,GAAG,GAAG;;;;;;;;;;;;8BAYU,iBAAS,kBAAkB,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BA4BzC,iBAAS,wBAAwB,YAAY;;;;;;;;8BAQ3C,iBAAS,wBAAwB,YAAY;;;;;;;;;;;SAWlE,CAAC;YAEF,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,WAAI,CAAC,gBAAgB,CAAC,WAAW,GAAG,IAAA,iCAAe,EAAC,6CAAqB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAE1G,MAAM,IAAI,CAAC,cAAc,CAAC,SAAU,CAAC,KAAK,CAAC,gBAAgB,iBAAS,KAAK,QAAQ,IAAI,GAAG,GAAG,EAAE;oBACzF,YAAY,EAAE,EAAE,MAAM,EAAE;iBAC3B,CAAC,CAAC;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACf,MAAM,GAAG,CAAC;YACd,CAAC;QACL,CAAC,CAAC;IAzGF,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,SAAU,CAAC;QAClD,MAAM,WAAW,GAAG,WAAI,CAAC,gBAAgB,CAAC,WAAW,GAAG,IAAA,iCAAe,EAAC,6CAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEhH,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,CACxC,uBAAuB,iBAAS,KAAK,WAAW,kCAAkC,EAClF,EAAE,IAAI,EAAE,sBAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;YAC5B,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,CACxC;iCACqB,iBAAS,MAAM,WAAW;;6BAE9B,EACjB,EAAE,IAAI,EAAE,sBAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;QACF,OAAO,WAAW,CAAC,UAAU,CAAC;IAClC,CAAC;CAgFJ;AAvHD,gEAuHC"}
|
|
@@ -49,6 +49,7 @@ const DeparturesDirectionRepository_1 = require("../workers/timetables/tasks/dat
|
|
|
49
49
|
const MetroRailtrackGPSRepository_1 = require("../workers/timetables/tasks/data-access/MetroRailtrackGPSRepository");
|
|
50
50
|
const GtfsStopsRedisRepository_1 = require("../workers/timetables/tasks/data-access/cache/GtfsStopsRedisRepository");
|
|
51
51
|
const PrecomputedTablesFacade_1 = require("../workers/timetables/tasks/helpers/PrecomputedTablesFacade");
|
|
52
|
+
const PrecomputedTablesLock_1 = require("../workers/timetables/tasks/helpers/PrecomputedTablesLock");
|
|
52
53
|
const GtfsFactory_1 = require("../workers/timetables/tasks/helpers/GtfsFactory");
|
|
53
54
|
const DeparturesDirectionTransformation_1 = require("../workers/timetables/tasks/transformations/DeparturesDirectionTransformation");
|
|
54
55
|
const MetroRailtrackDataTransformation_1 = require("../workers/timetables/tasks/transformations/MetroRailtrackDataTransformation");
|
|
@@ -89,6 +90,7 @@ GtfsContainer.registerSingleton(GtfsContainerToken_1.GtfsContainerToken.Departur
|
|
|
89
90
|
//#region Facade
|
|
90
91
|
GtfsContainer.registerSingleton(GtfsContainerToken_1.GtfsContainerToken.GtfsFacade, GtfsFacade_1.GtfsFacade);
|
|
91
92
|
GtfsContainer.registerSingleton(GtfsContainerToken_1.GtfsContainerToken.PrecomputedTablesFacade, PrecomputedTablesFacade_1.PrecomputedTablesFacade)
|
|
93
|
+
.registerSingleton(GtfsContainerToken_1.GtfsContainerToken.PrecomputedTablesLock, PrecomputedTablesLock_1.PrecomputedTablesLock)
|
|
92
94
|
.register(GtfsContainerToken_1.GtfsContainerToken.GtfsFactory, GtfsFactory_1.GtfsFactory, { lifecycle: tsyringe_1.Lifecycle.Transient })
|
|
93
95
|
.registerInstance(GtfsContainerToken_1.GtfsContainerToken.RedisPubSubChannel, new pubsub_1.RedisPubSubChannel(gtfs_1.Gtfs.name));
|
|
94
96
|
GtfsContainer.registerSingleton(GtfsContainerToken_1.GtfsContainerToken.DeparturePresetsFacade, DeparturePresetsFacade_1.DeparturePresetsFacade);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../../src/integration-engine/gtfs/ioc/Di.ts"],"names":[],"mappings":";;;AAAA,qCAA0C;AAC1C,8GAAyG;AACzG,gHAA2G;AAC3G,2DAAiC;AACjC,qFAA8F;AAC9F,iEAAoF;AACpF,4DAAyD;AACzD,sDAAmD;AACnD,8CAA2C;AAC3C,sEAAmE;AACnE,8EAA2E;AAC3E,kEAA+D;AAC/D,0EAAuE;AACvE,8FAA2F;AAC3F,oGAAiG;AACjG,4GAAyG;AACzG,sGAAmG;AACnG,8FAA2F;AAC3F,4DAAkE;AAClE,oFAAiF;AACjF,4HAAyH;AACzH,kHAA+G;AAC/G,wHAAqH;AACrH,6EAA0E;AAC1E,8GAA2G;AAC3G,8EAA2E;AAC3E,4GAAyG;AACzG,gHAA6G;AAC7G,4FAAyF;AACzF,0GAAuG;AACvG,4FAAyF;AACzF,sFAAmF;AACnF,yFAAsF;AACtF,uHAAoH;AACpH,2FAAwF;AACxF,iFAA8E;AAC9E,qGAAkG;AAClG,qGAAkG;AAClG,6GAA0G;AAC1G,2GAAwG;AACxG,yHAAsH;AACtH,yGAAsG;AACtG,uFAAoF;AACpF,mGAAgG;AAChG,yHAAsH;AACtH,qHAAkH;AAClH,qHAAkH;AAClH,yGAAsG;AACtG,iFAA8E;AAC9E,qIAAkI;AAClI,mIAAgI;AAChI,6DAA0D;AAE1D,wBAAwB;AACxB,MAAM,aAAa,GAAwB,iBAAY,CAAC,oBAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../../src/integration-engine/gtfs/ioc/Di.ts"],"names":[],"mappings":";;;AAAA,qCAA0C;AAC1C,8GAAyG;AACzG,gHAA2G;AAC3G,2DAAiC;AACjC,qFAA8F;AAC9F,iEAAoF;AACpF,4DAAyD;AACzD,sDAAmD;AACnD,8CAA2C;AAC3C,sEAAmE;AACnE,8EAA2E;AAC3E,kEAA+D;AAC/D,0EAAuE;AACvE,8FAA2F;AAC3F,oGAAiG;AACjG,4GAAyG;AACzG,sGAAmG;AACnG,8FAA2F;AAC3F,4DAAkE;AAClE,oFAAiF;AACjF,4HAAyH;AACzH,kHAA+G;AAC/G,wHAAqH;AACrH,6EAA0E;AAC1E,8GAA2G;AAC3G,8EAA2E;AAC3E,4GAAyG;AACzG,gHAA6G;AAC7G,4FAAyF;AACzF,0GAAuG;AACvG,4FAAyF;AACzF,sFAAmF;AACnF,yFAAsF;AACtF,uHAAoH;AACpH,2FAAwF;AACxF,iFAA8E;AAC9E,qGAAkG;AAClG,qGAAkG;AAClG,6GAA0G;AAC1G,2GAAwG;AACxG,yHAAsH;AACtH,yGAAsG;AACtG,uFAAoF;AACpF,mGAAgG;AAChG,yHAAsH;AACtH,qHAAkH;AAClH,qHAAkH;AAClH,yGAAsG;AACtG,qGAAkG;AAClG,iFAA8E;AAC9E,qIAAkI;AAClI,mIAAgI;AAChI,6DAA0D;AAE1D,wBAAwB;AACxB,MAAM,aAAa,GAAwB,iBAAY,CAAC,oBAAoB,EAAE,CAAC;AA4EtE,sCAAa;AA3EtB,YAAY;AAEZ,qBAAqB;AACrB,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,yBAAyB,EAAE,qDAAyB,CAAC,CAAC;AACzG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,gCAAgC,EAAE,mEAAgC,CAAC,CAAC;AACvH,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,2BAA2B,EAAE,yDAA2B,CAAC,CAAC;AAC7G,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,uBAAuB,EAAE,iDAAuB,CAAC,CAAC;AACrG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,gCAAgC,EAAE,mEAAgC,CAAC,CAAC;AACvH,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,qCAAqC,EAAE,6EAAqC,CAAC,CAAC;AACjI,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,mCAAmC,EAAE,yEAAmC,CAAC,CAAC;AAC7H,YAAY;AAEZ,sBAAsB;AACtB,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,yBAAyB,EAAE,qDAAyB,CAAC,CAAC,iBAAiB,CACtH,uCAAkB,CAAC,cAAc,EACjC,+BAAc,CACjB,CAAC;AACF,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,sBAAsB,EAAE,+CAAsB,CAAC,CAAC;AAC1F,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,kBAAkB,EAAE,uCAAkB,CAAC,CAAC;AAC3F,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,oBAAoB,EAAE,2CAAoB,CAAC,CAAC;AAC/F,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,mBAAmB,EAAE,kCAAoB,CAAC,CAAC;AACrF,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,6BAA6B,EAAE,6DAA6B,CAAC,CAAC;AACxG,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,cAAc,EAAE,+BAAc,CAAC,CAAC;AAC1E,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,gCAAgC,EAAE,mEAAgC,CAAC,CAAC;AAC9G,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,iCAAiC,EAAE,qEAAiC,CAAC,CAAC;AAChH,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,6BAA6B,EAAE,6DAA6B,CAAC,CAAC;AACxG,aAAa,CAAC,QAAQ,CAAC,uCAAkB,CAAC,iBAAiB,EAAE,qCAAiB,CAAC,CAAC;AAChF,YAAY;AAEZ,yBAAyB;AACzB,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,kBAAkB,EAAE,uCAAkB,CAAC,CAAC;AAC3F,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,kCAAkC,EAAE,uEAAkC,CAAC,CAAC;AAC3H,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,gCAAgC,EAAE,mEAAgC,CAAC,CAAC;AACvH,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,2BAA2B,EAAE,yDAA2B,CAAC,CAAC;AAC7G,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,iCAAiC,EAAE,qEAAiC,CAAC,CAAC;AACzH,YAAY;AAEZ,gBAAgB;AAChB,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,UAAU,EAAE,uBAAU,CAAC,CAAC;AAC3E,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,uBAAuB,EAAE,iDAAuB,CAAC;KAC/F,iBAAiB,CAAC,uCAAkB,CAAC,qBAAqB,EAAE,6CAAqB,CAAC;KAClF,QAAQ,CAAC,uCAAkB,CAAC,WAAW,EAAE,yBAAW,EAAE,EAAE,SAAS,EAAE,oBAAS,CAAC,SAAS,EAAE,CAAC;KACzF,gBAAgB,CAAC,uCAAkB,CAAC,kBAAkB,EAAE,IAAI,2BAAkB,CAAC,WAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAEhG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,sBAAsB,EAAE,+CAAsB,CAAC,CAAC;AACnG,YAAY;AAEZ,eAAe;AACf,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,yBAAyB,EAAE,qDAAyB,CAAC,CAAC;AACzG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,6BAA6B,EAAE,6DAA6B,CAAC,CAAC;AACjH,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,mCAAmC,EAAE,yEAAmC,CAAC,CAAC;AAC7H,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,2BAA2B,EAAE,yDAA2B,CAAC,CAAC;AAC7G,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,6BAA6B,EAAE,6DAA6B,CAAC,CAAC;AACjH,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,gCAAgC,EAAE,mEAAgC,CAAC,CAAC;AACvH,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,kBAAkB,EAAE,uCAAkB,CAAC,CAAC;AAC3F,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,eAAe,EAAE,iCAAe,CAAC,CAAC;AACrF,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,oBAAoB,EAAE,2CAAoB,CAAC,CAAC;AAC/F,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,yBAAyB,EAAE,qDAAyB,CAAC,CAAC;AACzG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,mBAAmB,EAAE,yCAAmB,CAAC,CAAC;AAC7F,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,kCAAkC,EAAE,uEAAkC,CAAC,CAAC;AAC3H,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,4BAA4B,EAAE,2DAA4B,CAAC,CAAC;AAC/G,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,wBAAwB,EAAE,mDAAwB,CAAC,CAAC;AACvG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,sBAAsB,EAAE,+CAAsB,CAAC,CAAC;AACnG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,sBAAsB,EAAE,+CAAsB,CAAC,CAAC;AACnG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,mBAAmB,EAAE,yCAAmB,CAAC,CAAC;AAC7F,YAAY;AAEZ,iBAAiB;AACjB,YAAY;AAEZ,eAAe;AACf,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,wBAAwB,EAAE,mDAAwB,CAAC,CAAC;AACvG,aAAa,CAAC,iBAAiB,CAAC,uCAAkB,CAAC,4BAA4B,EAAE,2DAA4B,CAAC,CAAC"}
|
|
@@ -5,6 +5,7 @@ declare const GtfsContainerToken: {
|
|
|
5
5
|
GtfsFeedRepository: symbol;
|
|
6
6
|
GtfsRoutesRepository: symbol;
|
|
7
7
|
PrecomputedTablesFacade: symbol;
|
|
8
|
+
PrecomputedTablesLock: symbol;
|
|
8
9
|
StaticFileRedisRepository: symbol;
|
|
9
10
|
GtfsTransformation: symbol;
|
|
10
11
|
PublicDepartureCacheTransformation: symbol;
|
|
@@ -8,6 +8,7 @@ const GtfsContainerToken = {
|
|
|
8
8
|
GtfsFeedRepository: Symbol("GtfsFeedRepository"),
|
|
9
9
|
GtfsRoutesRepository: Symbol("GtfsRoutesRepository"),
|
|
10
10
|
PrecomputedTablesFacade: Symbol("PrecomputedTablesFacade"),
|
|
11
|
+
PrecomputedTablesLock: Symbol("PrecomputedTablesLock"),
|
|
11
12
|
StaticFileRedisRepository: Symbol("StaticFileRedisRepository"),
|
|
12
13
|
GtfsTransformation: Symbol("GtfsTransformation"),
|
|
13
14
|
PublicDepartureCacheTransformation: Symbol("PublicDepartureCacheTransformation"),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GtfsContainerToken.js","sourceRoot":"","sources":["../../../../src/integration-engine/gtfs/ioc/GtfsContainerToken.ts"],"names":[],"mappings":";;;AAAA,MAAM,kBAAkB,GAAG;IACvB,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,CAAC;IACpD,uBAAuB,EAAE,MAAM,CAAC,yBAAyB,CAAC;IAC1D,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,kCAAkC,EAAE,MAAM,CAAC,oCAAoC,CAAC;IAChF,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,cAAc,EAAE,MAAM,CAAC,gBAAgB,CAAC;IACxC,mBAAmB,EAAE,MAAM,CAAC,qBAAqB,CAAC;IAClD,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,mCAAmC,EAAE,MAAM,CAAC,qCAAqC,CAAC;IAClF,2BAA2B,EAAE,MAAM,CAAC,6BAA6B,CAAC;IAClE,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,cAAc,EAAE,MAAM,CAAC,gBAAgB,CAAC;IACxC,uBAAuB,EAAE,MAAM,CAAC,yBAAyB,CAAC;IAC1D,uBAAuB,EAAE,MAAM,CAAC,yBAAyB,CAAC;IAC1D,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,2BAA2B,EAAE,MAAM,CAAC,6BAA6B,CAAC;IAClE,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,qCAAqC,EAAE,MAAM,CAAC,uCAAuC,CAAC;IACtF,mCAAmC,EAAE,MAAM,CAAC,qCAAqC,CAAC;IAClF,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,iCAAiC,EAAE,MAAM,CAAC,mCAAmC,CAAC;IAC9E,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,wBAAwB,EAAE,MAAM,CAAC,0BAA0B,CAAC;IAC5D,4BAA4B,EAAE,MAAM,CAAC,8BAA8B,CAAC;IACpE,iBAAiB,EAAE,MAAM,CAAC,mBAAmB,CAAC;IAC9C,eAAe,EAAE,MAAM,CAAC,iBAAiB,CAAC;IAC1C,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,CAAC;IACpD,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,mBAAmB,EAAE,MAAM,CAAC,qBAAqB,CAAC;IAClD,kCAAkC,EAAE,MAAM,CAAC,oCAAoC,CAAC;IAChF,4BAA4B,EAAE,MAAM,CAAC,8BAA8B,CAAC;IACpE,wBAAwB,EAAE,MAAM,CAAC,0BAA0B,CAAC;IAC5D,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,mBAAmB,EAAE,MAAM,CAAC,qBAAqB,CAAC;IAClD,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,2BAA2B,EAAE,MAAM,CAAC,6BAA6B,CAAC;IAClE,iCAAiC,EAAE,MAAM,CAAC,mCAAmC,CAAC;CACjF,CAAC;AAEO,gDAAkB"}
|
|
1
|
+
{"version":3,"file":"GtfsContainerToken.js","sourceRoot":"","sources":["../../../../src/integration-engine/gtfs/ioc/GtfsContainerToken.ts"],"names":[],"mappings":";;;AAAA,MAAM,kBAAkB,GAAG;IACvB,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,CAAC;IACpD,uBAAuB,EAAE,MAAM,CAAC,yBAAyB,CAAC;IAC1D,qBAAqB,EAAE,MAAM,CAAC,uBAAuB,CAAC;IACtD,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,kCAAkC,EAAE,MAAM,CAAC,oCAAoC,CAAC;IAChF,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,cAAc,EAAE,MAAM,CAAC,gBAAgB,CAAC;IACxC,mBAAmB,EAAE,MAAM,CAAC,qBAAqB,CAAC;IAClD,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,mCAAmC,EAAE,MAAM,CAAC,qCAAqC,CAAC;IAClF,2BAA2B,EAAE,MAAM,CAAC,6BAA6B,CAAC;IAClE,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,cAAc,EAAE,MAAM,CAAC,gBAAgB,CAAC;IACxC,uBAAuB,EAAE,MAAM,CAAC,yBAAyB,CAAC;IAC1D,uBAAuB,EAAE,MAAM,CAAC,yBAAyB,CAAC;IAC1D,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,2BAA2B,EAAE,MAAM,CAAC,6BAA6B,CAAC;IAClE,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,qCAAqC,EAAE,MAAM,CAAC,uCAAuC,CAAC;IACtF,mCAAmC,EAAE,MAAM,CAAC,qCAAqC,CAAC;IAClF,kBAAkB,EAAE,MAAM,CAAC,oBAAoB,CAAC;IAChD,iCAAiC,EAAE,MAAM,CAAC,mCAAmC,CAAC;IAC9E,gCAAgC,EAAE,MAAM,CAAC,kCAAkC,CAAC;IAC5E,wBAAwB,EAAE,MAAM,CAAC,0BAA0B,CAAC;IAC5D,4BAA4B,EAAE,MAAM,CAAC,8BAA8B,CAAC;IACpE,iBAAiB,EAAE,MAAM,CAAC,mBAAmB,CAAC;IAC9C,eAAe,EAAE,MAAM,CAAC,iBAAiB,CAAC;IAC1C,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,CAAC;IACpD,yBAAyB,EAAE,MAAM,CAAC,2BAA2B,CAAC;IAC9D,mBAAmB,EAAE,MAAM,CAAC,qBAAqB,CAAC;IAClD,kCAAkC,EAAE,MAAM,CAAC,oCAAoC,CAAC;IAChF,4BAA4B,EAAE,MAAM,CAAC,8BAA8B,CAAC;IACpE,wBAAwB,EAAE,MAAM,CAAC,0BAA0B,CAAC;IAC5D,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,sBAAsB,EAAE,MAAM,CAAC,wBAAwB,CAAC;IACxD,mBAAmB,EAAE,MAAM,CAAC,qBAAqB,CAAC;IAClD,6BAA6B,EAAE,MAAM,CAAC,+BAA+B,CAAC;IACtE,2BAA2B,EAAE,MAAM,CAAC,6BAA6B,CAAC;IAClE,iCAAiC,EAAE,MAAM,CAAC,mCAAmC,CAAC;CACjF,CAAC;AAEO,gDAAkB"}
|
|
@@ -6,12 +6,14 @@ import { ISimpleConfig } from "@golemio/core/dist/helpers/configuration/ISimpleC
|
|
|
6
6
|
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
7
7
|
import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
|
|
8
8
|
import { PrecomputedTablesFacade } from "./helpers/PrecomputedTablesFacade";
|
|
9
|
+
import { PrecomputedTablesLock } from "./helpers/PrecomputedTablesLock";
|
|
9
10
|
import { IDatasetsInput } from "./interfaces/IDatasetsInput";
|
|
10
11
|
import { DatasetsInputSchema } from "./schema/DownloadDataInputSchema";
|
|
11
12
|
export declare class CheckSavedRowsAndReplaceTablesTask extends AbstractTask<IDatasetsInput> {
|
|
12
13
|
private readonly metadataRepository;
|
|
13
14
|
private readonly gtfsRepository;
|
|
14
15
|
private readonly precomputeTablesFacade;
|
|
16
|
+
private readonly precomputedTablesLock;
|
|
15
17
|
private readonly gtfsFacade;
|
|
16
18
|
private readonly feedRepository;
|
|
17
19
|
private readonly config;
|
|
@@ -20,7 +22,7 @@ export declare class CheckSavedRowsAndReplaceTablesTask extends AbstractTask<IDa
|
|
|
20
22
|
readonly queueTtl: number;
|
|
21
23
|
readonly schema: typeof DatasetsInputSchema;
|
|
22
24
|
private readonly dataCacheManager;
|
|
23
|
-
constructor(metadataRepository: GtfsMetadataRepository, gtfsRepository: GtfsRepository, precomputeTablesFacade: PrecomputedTablesFacade, gtfsFacade: GtfsFacade, feedRepository: GtfsFeedRepository, config: ISimpleConfig, logger: ILogger);
|
|
25
|
+
constructor(metadataRepository: GtfsMetadataRepository, gtfsRepository: GtfsRepository, precomputeTablesFacade: PrecomputedTablesFacade, precomputedTablesLock: PrecomputedTablesLock, gtfsFacade: GtfsFacade, feedRepository: GtfsFeedRepository, config: ISimpleConfig, logger: ILogger);
|
|
24
26
|
protected execute(data: IDatasetsInput): Promise<void>;
|
|
25
27
|
private retryOrThrow;
|
|
26
28
|
private checkSavedRows;
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.js
CHANGED
|
@@ -29,13 +29,15 @@ const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
|
|
|
29
29
|
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
30
30
|
const constants_2 = require("../constants");
|
|
31
31
|
const PrecomputedTablesFacade_1 = require("./helpers/PrecomputedTablesFacade");
|
|
32
|
+
const PrecomputedTablesLock_1 = require("./helpers/PrecomputedTablesLock");
|
|
32
33
|
const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
|
|
33
34
|
let CheckSavedRowsAndReplaceTablesTask = class CheckSavedRowsAndReplaceTablesTask extends workers_1.AbstractTask {
|
|
34
|
-
constructor(metadataRepository, gtfsRepository, precomputeTablesFacade, gtfsFacade, feedRepository, config, logger) {
|
|
35
|
+
constructor(metadataRepository, gtfsRepository, precomputeTablesFacade, precomputedTablesLock, gtfsFacade, feedRepository, config, logger) {
|
|
35
36
|
super(constants_2.TIMETABLE_WORKER_NAME);
|
|
36
37
|
this.metadataRepository = metadataRepository;
|
|
37
38
|
this.gtfsRepository = gtfsRepository;
|
|
38
39
|
this.precomputeTablesFacade = precomputeTablesFacade;
|
|
40
|
+
this.precomputedTablesLock = precomputedTablesLock;
|
|
39
41
|
this.gtfsFacade = gtfsFacade;
|
|
40
42
|
this.feedRepository = feedRepository;
|
|
41
43
|
this.config = config;
|
|
@@ -60,11 +62,13 @@ let CheckSavedRowsAndReplaceTablesTask = class CheckSavedRowsAndReplaceTablesTas
|
|
|
60
62
|
const allSaved = await this.metadataRepository.checkAllTablesHasSavedState(notDeployedVersions, feedCode);
|
|
61
63
|
try {
|
|
62
64
|
await this.checkSavedRows(allSaved, datasetDeployInfo, feedCode);
|
|
63
|
-
await this.
|
|
64
|
-
|
|
65
|
-
|
|
65
|
+
await this.precomputedTablesLock.runExclusive(async () => {
|
|
66
|
+
await this.precomputeTablesFacade.createAndPopulatePrecomputedTmpTables(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Tmp, feedCode);
|
|
67
|
+
const feedId = await this.feedRepository.getFeedIdByCode(feedCode);
|
|
68
|
+
await this.gtfsFacade.replaceTables(notDeployedVersions, feedCode, feedId);
|
|
69
|
+
await this.gtfsRepository.cleanTmpAndOldTables(feedCode);
|
|
70
|
+
});
|
|
66
71
|
await this.dataCacheManager.cleanCache();
|
|
67
|
-
await this.gtfsRepository.cleanTmpAndOldTables(feedCode);
|
|
68
72
|
const rabbitExchangeName = this.config.getValue("env.RABBIT_EXCHANGE_NAME");
|
|
69
73
|
// Re-point VP trips whose gtfs_trip_id vanished with the replaced timetable. The empty payload
|
|
70
74
|
// is intentional: the downstream chain (RefreshGtfsTripDataTask -> HttpGtfsTripDataFixer)
|
|
@@ -96,7 +100,8 @@ let CheckSavedRowsAndReplaceTablesTask = class CheckSavedRowsAndReplaceTablesTas
|
|
|
96
100
|
const retries = await Promise.all(notDeployedVersions.map(async (el) => {
|
|
97
101
|
return await this.metadataRepository.getNumberOfDownloadRetries(el.dataset, el.version, feedCode);
|
|
98
102
|
}));
|
|
99
|
-
|
|
103
|
+
// under the lock: the cleanup drops the feed's precomputed _tmp_ tables a refresh may be filling
|
|
104
|
+
await this.precomputedTablesLock.runExclusive(() => this.gtfsRepository.cleanTmpAndOldTables(feedCode));
|
|
100
105
|
if (Math.max(...retries) <= 5) {
|
|
101
106
|
// re-acquire the source data and try again. The FTP download path only ever serves the
|
|
102
107
|
// PID feed -- retrying a storage-imported feed through it would download PID's data and
|
|
@@ -110,8 +115,8 @@ let CheckSavedRowsAndReplaceTablesTask = class CheckSavedRowsAndReplaceTablesTas
|
|
|
110
115
|
integration_engine_1.IntegrationErrorHandler.handle(new golemio_errors_1.GeneralError(`Error while checking Gtfs saved rows. Attempt number ${Math.min(...retries)} was resent.`, this.constructor.name, err));
|
|
111
116
|
}
|
|
112
117
|
else {
|
|
113
|
-
// at least refresh precomputed tables and finish with error
|
|
114
|
-
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
|
|
118
|
+
// at least refresh the feed's precomputed tables from its current timetable and finish with error
|
|
119
|
+
integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", { feedCode });
|
|
115
120
|
throw new golemio_errors_1.GeneralError("Error while checking Gtfs saved rows.", this.constructor.name, err);
|
|
116
121
|
}
|
|
117
122
|
}
|
|
@@ -141,13 +146,15 @@ exports.CheckSavedRowsAndReplaceTablesTask = CheckSavedRowsAndReplaceTablesTask
|
|
|
141
146
|
__param(0, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsMetadataRepository)),
|
|
142
147
|
__param(1, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsRepository)),
|
|
143
148
|
__param(2, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.PrecomputedTablesFacade)),
|
|
144
|
-
__param(3, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.
|
|
145
|
-
__param(4, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.
|
|
146
|
-
__param(5, (0, tsyringe_1.inject)(
|
|
147
|
-
__param(6, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.
|
|
149
|
+
__param(3, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.PrecomputedTablesLock)),
|
|
150
|
+
__param(4, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsFacade)),
|
|
151
|
+
__param(5, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsFeedRepository)),
|
|
152
|
+
__param(6, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.SimpleConfig)),
|
|
153
|
+
__param(7, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
|
|
148
154
|
__metadata("design:paramtypes", [GtfsMetadataRepository_1.GtfsMetadataRepository,
|
|
149
155
|
GtfsRepository_1.GtfsRepository,
|
|
150
156
|
PrecomputedTablesFacade_1.PrecomputedTablesFacade,
|
|
157
|
+
PrecomputedTablesLock_1.PrecomputedTablesLock,
|
|
151
158
|
GtfsFacade_1.GtfsFacade,
|
|
152
159
|
GtfsFeedRepository_1.GtfsFeedRepository, Object, Object])
|
|
153
160
|
], CheckSavedRowsAndReplaceTablesTask);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CheckSavedRowsAndReplaceTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,wEAAgD;AAChD,oDAAiD;AACjD,4EAAyE;AACzE,gFAA6E;AAC7E,wEAAqE;AACrE,wEAAqE;AACrE,kFAA+E;AAC/E,wEAAqE;AAErE,iGAA0G;AAE1G,wEAAqE;AAErE,8EAA8F;AAC9F,2EAA6E;AAC7E,6EAAwE;AACxE,iEAAwE;AACxE,4CAAqD;AAErD,+EAA4E;
|
|
1
|
+
{"version":3,"file":"CheckSavedRowsAndReplaceTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/CheckSavedRowsAndReplaceTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,wEAAgD;AAChD,oDAAiD;AACjD,4EAAyE;AACzE,gFAA6E;AAC7E,wEAAqE;AACrE,wEAAqE;AACrE,kFAA+E;AAC/E,wEAAqE;AAErE,iGAA0G;AAE1G,wEAAqE;AAErE,8EAA8F;AAC9F,2EAA6E;AAC7E,6EAAwE;AACxE,iEAAwE;AACxE,4CAAqD;AAErD,+EAA4E;AAC5E,2EAAwE;AAExE,8EAAuE;AAGhE,IAAM,kCAAkC,GAAxC,MAAM,kCAAmC,SAAQ,sBAA4B;IAOhF,YAEI,kBAA2D,EAChB,cAA+C,EACtC,sBAAgE,EAClE,qBAA6D,EACxE,UAAuC,EAC/B,cAAmD,EAClE,MAAsC,EAC5C,MAAgC;QAE1D,KAAK,CAAC,iCAAqB,CAAC,CAAC;QATZ,uBAAkB,GAAlB,kBAAkB,CAAwB;QACC,mBAAc,GAAd,cAAc,CAAgB;QACrB,2BAAsB,GAAtB,sBAAsB,CAAyB;QACjD,0BAAqB,GAArB,qBAAqB,CAAuB;QACvD,eAAU,GAAV,UAAU,CAAY;QACd,mBAAc,GAAd,cAAc,CAAoB;QACjD,WAAM,GAAN,MAAM,CAAe;QAC3B,WAAM,GAAN,MAAM,CAAS;QAf9C,cAAS,GAAG,gCAAgC,CAAC;QAC7C,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAC3C,WAAM,GAAG,6CAAmB,CAAC;QAE5B,qBAAgB,GAAqB,mCAAgB,CAAC,WAAW,EAAE,CAAC;IAcrF,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,IAAoB;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,qBAAa,CAAC;QAChD,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAEhF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,KAAK,SAAS,IAAI,CAAC,QAAQ,oBAAoB,CAAC,CAAC;QAC1H,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,mBAAmB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,QAAQ,CAAC,CAAC;QAE1G,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,CAAC,CAAC;YACjE,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;gBACrD,MAAM,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,6CAAqB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC7G,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;gBACnE,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,mBAAmB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC3E,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;YAEzC,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAS,0BAA0B,CAAC,CAAC;YAEpF,+FAA+F;YAC/F,0FAA0F;YAC1F,0FAA0F;YAC1F,8FAA8F;YAC9F,MAAM,iCAAY,CAAC,qBAAqB,CACpC,GAAG,kBAAkB,IAAI,uBAAc,CAAC,WAAW,EAAE,EAAE,EACvD,qBAAqB,EACrB,EAAE,CACL,CAAC;YAEF,sFAAsF;YACtF,kGAAkG;YAClG,8FAA8F;YAC9F,oFAAoF;YACpF,sBAAsB;YACtB,mGAAmG;YACnG,yFAAyF;YACzF,6FAA6F;YAC7F,2FAA2F;YAC3F,MAAM;QACV,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CACtB,mBAAsE,EACtE,IAAoB,EACpB,GAAQ;QAER,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,qBAAa,CAAC;QAChD,4BAA4B;QAC5B,MAAM,OAAO,CAAC,GAAG,CACb,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACjC,MAAM,IAAI,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACzF,CAAC,CAAC,CACL,CAAC;QACF,wBAAwB;QACxB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACjC,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,0BAA0B,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACtG,CAAC,CAAC,CACL,CAAC;QAEF,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAExG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,uFAAuF;YACvF,wFAAwF;YACxF,wFAAwF;YACxF,IAAI,QAAQ,KAAK,qBAAa,EAAE,CAAC;gBAC7B,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,EAAE,IAAI,CAAC,CAAC;YACnF,CAAC;iBAAM,CAAC;gBACJ,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,uBAAuB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;YAChG,CAAC;YAED,4CAAuB,CAAC,MAAM,CAC1B,IAAI,6BAAY,CACZ,wDAAwD,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,cAAc,EAC1F,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CACJ,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,kGAAkG;YAClG,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE/F,MAAM,IAAI,6BAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAChG,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,cAAc,CACxB,QAAiB,EACjB,iBAIE,EACF,QAAgB;QAEhB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,IAAI,6BAAY,CAAC,0CAA0C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CACb,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC7F,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,IAAoB;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,qBAAa,CAAC;QAChD,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAChC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACtF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,kCAAkC,CAC/E,OAAO,EACP,YAAY,CAAC,OAAO,EACpB,QAAQ,CACX,CAAC;YACF,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;QACjD,CAAC,CAAC,CACL,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAE3E,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzE,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,CAAC;IAC9C,CAAC;CACJ,CAAA;AAjKY,gFAAkC;6CAAlC,kCAAkC;IAD9C,IAAA,qBAAU,GAAE;IASJ,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,cAAc,CAAC,CAAA;IACzC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,uBAAuB,CAAC,CAAA;IAClD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,qBAAqB,CAAC,CAAA;IAChD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,UAAU,CAAC,CAAA;IACrC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,kBAAkB,CAAC,CAAA;IAC7C,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,YAAY,CAAC,CAAA;IAC9B,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;qCAPY,+CAAsB;QACiB,+BAAc;QACG,iDAAuB;QAC1B,6CAAqB;QAC3C,uBAAU;QACE,uCAAkB;GAd7F,kCAAkC,CAiK9C"}
|
|
@@ -105,7 +105,7 @@ let DownloadDatasetsTask = class DownloadDatasetsTask extends workers_1.Abstract
|
|
|
105
105
|
cachedDatasets = cachedDatasets.concat(await this.downloadAndPersistDatasetFiles(dataset, feedCode));
|
|
106
106
|
}
|
|
107
107
|
catch (err) {
|
|
108
|
-
await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
|
|
108
|
+
await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", { feedCode });
|
|
109
109
|
if (err instanceof golemio_errors_1.AbstractGolemioError) {
|
|
110
110
|
throw err;
|
|
111
111
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DownloadDatasetsTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/DownloadDatasetsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,4EAAyE;AACzE,wEAAqE;AACrE,oGAAiG;AACjG,8DAA2D;AAC3D,wEAAqE;AACrE,8EAAoG;AACpG,wEAAqE;AAErE,8EAA4E;AAC5E,qFAA8F;AAC9F,2EAA6E;AAC7E,6EAA8F;AAC9F,iEAAwE;AAExE,sDAAyB;AACzB,uDAAkH;AAClH,uDAAoD;AAEpD,8EAAuE;AACvE,4CAAqD;AAG9C,IAAM,oBAAoB,GAA1B,MAAM,oBAAqB,SAAQ,sBAA4B;IAKlE,YAEI,kBAA2D,EAE3D,yBAAqE,EAC1B,cAA+C,EAClD,WAAyC,EAClC,gBAAqD,EAC1E,MAAgC;QAE1D,KAAK,CAAC,iCAAqB,CAAC,CAAC;QARZ,uBAAkB,GAAlB,kBAAkB,CAAwB;QAE1C,8BAAyB,GAAzB,yBAAyB,CAA2B;QACT,mBAAc,GAAd,cAAc,CAAgB;QACjC,gBAAW,GAAX,WAAW,CAAa;QACjB,qBAAgB,GAAhB,gBAAgB,CAAoB;QACzD,WAAM,GAAN,MAAM,CAAS;QAZ9C,cAAS,GAAG,kBAAkB,CAAC;QAC/B,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QACxC,WAAM,GAAG,6CAAmB,CAAC;QA6E7C;;WAEG;QACK,mCAA8B,GAAG,KAAK,EAAE,OAAqB,EAAE,QAAgB,EAA6B,EAAE;YAClH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;YACpD,IAAI,KAAK,GAAoB,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACjG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;YAE3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACxF,MAAM,aAAa,GAAqB,EAAE,CAAC;YAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC7D,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;gBAC/B,IAAI,OAAO,KAAK,yBAAW,CAAC,IAAI,IAAI,OAAO,KAAK,yBAAW,CAAC,WAAW,EAAE,CAAC;oBACtE,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAkB,CAAC,CAAC;gBACvF,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBACpH,CAAC;gBAED,MAAM,GAAG,GAAG,IAAA,iCAAmB,EAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC/D,OAAO;wBACH,OAAO;wBACP,GAAG,EAAE,IAAI;wBACT,IAAI,EAAE,iCAAY,CAAC,KAAK;wBACxB,KAAK,EAAE,kCAAa,CAAC,UAAU;wBAC/B,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;qBACtC,CAAC;gBACN,CAAC,CAAC,CAAC;gBAEH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACjC,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACvE,CAAC;gBAED,aAAa,CAAC,IAAI,CAAC;oBACf,GAAG,IAAI;oBACP,OAAO;oBACP,QAAQ;iBACX,CAAC,CAAC;YACP,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,OAAuB,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;YAElH,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAC9B;gBACI,OAAO;gBACP,GAAG,EAAE,2CAAsB,CAAC,aAAa;gBACzC,IAAI,EAAE,iCAAY,CAAC,YAAY;gBAC/B,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;aACtC,EACD,SAAS,EACT,QAAQ,CACX,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAChG,OAAO,aAAa,CAAC;QACzB,CAAC,CAAC;IA1HF,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,IAAoB;QACxC,iFAAiF;QACjF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;QACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,QAAQ,UAAU,QAAQ,GAAG,CAAC,CAAC;QAE9E,IAAI,cAAc,GAAqB,EAAE,CAAC;QAE1C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,IAAI,CAAC;gBACD,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,8BAA8B,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YACzG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"DownloadDatasetsTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/DownloadDatasetsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,4EAAyE;AACzE,wEAAqE;AACrE,oGAAiG;AACjG,8DAA2D;AAC3D,wEAAqE;AACrE,8EAAoG;AACpG,wEAAqE;AAErE,8EAA4E;AAC5E,qFAA8F;AAC9F,2EAA6E;AAC7E,6EAA8F;AAC9F,iEAAwE;AAExE,sDAAyB;AACzB,uDAAkH;AAClH,uDAAoD;AAEpD,8EAAuE;AACvE,4CAAqD;AAG9C,IAAM,oBAAoB,GAA1B,MAAM,oBAAqB,SAAQ,sBAA4B;IAKlE,YAEI,kBAA2D,EAE3D,yBAAqE,EAC1B,cAA+C,EAClD,WAAyC,EAClC,gBAAqD,EAC1E,MAAgC;QAE1D,KAAK,CAAC,iCAAqB,CAAC,CAAC;QARZ,uBAAkB,GAAlB,kBAAkB,CAAwB;QAE1C,8BAAyB,GAAzB,yBAAyB,CAA2B;QACT,mBAAc,GAAd,cAAc,CAAgB;QACjC,gBAAW,GAAX,WAAW,CAAa;QACjB,qBAAgB,GAAhB,gBAAgB,CAAoB;QACzD,WAAM,GAAN,MAAM,CAAS;QAZ9C,cAAS,GAAG,kBAAkB,CAAC;QAC/B,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QACxC,WAAM,GAAG,6CAAmB,CAAC;QA6E7C;;WAEG;QACK,mCAA8B,GAAG,KAAK,EAAE,OAAqB,EAAE,QAAgB,EAA6B,EAAE;YAClH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;YACpD,IAAI,KAAK,GAAoB,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACjG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;YAE3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACxF,MAAM,aAAa,GAAqB,EAAE,CAAC;YAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC7D,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;gBAC/B,IAAI,OAAO,KAAK,yBAAW,CAAC,IAAI,IAAI,OAAO,KAAK,yBAAW,CAAC,WAAW,EAAE,CAAC;oBACtE,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAkB,CAAC,CAAC;gBACvF,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBACpH,CAAC;gBAED,MAAM,GAAG,GAAG,IAAA,iCAAmB,EAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC/D,OAAO;wBACH,OAAO;wBACP,GAAG,EAAE,IAAI;wBACT,IAAI,EAAE,iCAAY,CAAC,KAAK;wBACxB,KAAK,EAAE,kCAAa,CAAC,UAAU;wBAC/B,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;qBACtC,CAAC;gBACN,CAAC,CAAC,CAAC;gBAEH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACjC,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACvE,CAAC;gBAED,aAAa,CAAC,IAAI,CAAC;oBACf,GAAG,IAAI;oBACP,OAAO;oBACP,QAAQ;iBACX,CAAC,CAAC;YACP,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,OAAuB,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;YAElH,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAC9B;gBACI,OAAO;gBACP,GAAG,EAAE,2CAAsB,CAAC,aAAa;gBACzC,IAAI,EAAE,iCAAY,CAAC,YAAY;gBAC/B,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,CAAC;aACtC,EACD,SAAS,EACT,QAAQ,CACX,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAChG,OAAO,aAAa,CAAC;QACzB,CAAC,CAAC;IA1HF,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,IAAoB;QACxC,iFAAiF;QACjF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;QACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,QAAQ,UAAU,QAAQ,GAAG,CAAC,CAAC;QAE9E,IAAI,cAAc,GAAqB,EAAE,CAAC;QAE1C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,IAAI,CAAC;gBACD,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,8BAA8B,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YACzG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,0BAA0B,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAErG,IAAI,GAAG,YAAY,qCAAoB,EAAE,CAAC;oBACtC,MAAM,GAAG,CAAC;gBACd,CAAC;gBAED,MAAM,IAAI,6BAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAChG,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;YACtD,aAAa,EAAE,iBAAE,CAAC,QAAQ,EAAE;YAC5B,eAAe,EAAE,cAAc,CAAC,MAAM;SACzC,CAAC,CAAC;QACH,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAE7B,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;YACzC,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE;gBACxF,OAAO,EAAE;oBACL,CAAC,oCAAsB,CAAC,EAAE,iBAAE,CAAC,QAAQ,EAAE;iBAC1C;aACJ,CAAC,CAAC;QACP,CAAC;QAED,sEAAsE;QACtE,wDAAwD;QACxD,IAAI,CAAC;YACD,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,EAAE;gBAC3D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,OAAO,KAAK,YAAY,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;gBAEvG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,OAAO,EAAE,CAAC,CAAC;oBAC9D,MAAM,IAAI,6BAAY,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC3D,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;gBACxD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,qCAAoB,EAAE,CAAC;gBACtC,MAAM,GAAG,CAAC;YACd,CAAC;YACD,MAAM,IAAI,6BAAY,CAAC,oCAAoC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7F,CAAC;gBAAS,CAAC;YACP,MAAM,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACzB,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,gCAAgC,EAAE,IAAI,CAAC,CAAC;QACjG,CAAC;IACL,CAAC;CA6DJ,CAAA;AA3IY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,qBAAU,GAAE;IAOJ,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,yBAAyB,CAAC,CAAA;IAEpD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,cAAc,CAAC,CAAA;IACzC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,WAAW,CAAC,CAAA;IACtC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,kBAAkB,CAAC,CAAA;IAC7C,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;qCANY,+CAAsB;QAEf,qDAAyB;QACO,+BAAc;QACpB,yBAAW;QACC,2BAAkB;GAZ/F,oBAAoB,CA2IhC"}
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.d.ts
CHANGED
|
@@ -2,18 +2,36 @@ import { GtfsFacade } from "../../../GtfsFacade";
|
|
|
2
2
|
import { GtfsFeedRepository } from "../../../data-access/GtfsFeedRepository";
|
|
3
3
|
import { GtfsRepository } from "../../../data-access/GtfsRepository";
|
|
4
4
|
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
5
|
-
import {
|
|
5
|
+
import { AbstractTask } from "@golemio/core/dist/integration-engine/workers";
|
|
6
6
|
import { PrecomputedTablesFacade } from "./helpers/PrecomputedTablesFacade";
|
|
7
|
-
|
|
7
|
+
import { PrecomputedTablesLock } from "./helpers/PrecomputedTablesLock";
|
|
8
|
+
import { IRefreshPrecomputedTablesInput } from "./interfaces/IRefreshPrecomputedTablesInput";
|
|
9
|
+
import { RefreshPrecomputedTablesSchema } from "./schema/RefreshPrecomputedTablesSchema";
|
|
10
|
+
/**
|
|
11
|
+
* Rebuilds the precomputed tables from the live (actual) GTFS tables. They hold a rolling date window
|
|
12
|
+
* (see ServicesCalendarRepository), so every feed needs a rebuild once a day even when its timetable
|
|
13
|
+
* did not change.
|
|
14
|
+
* - without feedCode (daily cron): picks the feeds that have data and were not rebuilt today yet (e.g. by
|
|
15
|
+
* an import) and starts a chain -- one message per feed, each sending the next one when done, so no
|
|
16
|
+
* single message stays unacknowledged for the whole run
|
|
17
|
+
* - with feedCode: rebuilds that feed, then passes the chain on (a failing feed does not stop it); the
|
|
18
|
+
* import failure paths send a lone feedCode, which is a chain of one
|
|
19
|
+
*/
|
|
20
|
+
export declare class RefreshPrecomputedTablesTask extends AbstractTask<IRefreshPrecomputedTablesInput> {
|
|
8
21
|
private readonly gtfsRepository;
|
|
9
22
|
private readonly precomputeTablesFacade;
|
|
23
|
+
private readonly precomputedTablesLock;
|
|
10
24
|
private readonly gtfsFacade;
|
|
11
25
|
private readonly feedRepository;
|
|
12
26
|
private readonly logger;
|
|
13
27
|
readonly queueName = "refreshPrecomputedTables";
|
|
14
28
|
readonly queueTtl: number;
|
|
29
|
+
readonly schema: typeof RefreshPrecomputedTablesSchema;
|
|
15
30
|
private readonly dataCacheManager;
|
|
16
|
-
constructor(gtfsRepository: GtfsRepository, precomputeTablesFacade: PrecomputedTablesFacade, gtfsFacade: GtfsFacade, feedRepository: GtfsFeedRepository, logger: ILogger);
|
|
17
|
-
protected execute(): Promise<void>;
|
|
18
|
-
private
|
|
31
|
+
constructor(gtfsRepository: GtfsRepository, precomputeTablesFacade: PrecomputedTablesFacade, precomputedTablesLock: PrecomputedTablesLock, gtfsFacade: GtfsFacade, feedRepository: GtfsFeedRepository, logger: ILogger);
|
|
32
|
+
protected execute(data: IRefreshPrecomputedTablesInput): Promise<void>;
|
|
33
|
+
private startChain;
|
|
34
|
+
private sendRefresh;
|
|
35
|
+
private refreshFeed;
|
|
36
|
+
private rebuildFeed;
|
|
19
37
|
}
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js
CHANGED
|
@@ -20,50 +20,113 @@ const DataCacheManager_1 = require("../../../helpers/DataCacheManager");
|
|
|
20
20
|
const SourceTableSuffixEnum_1 = require("../../../helpers/SourceTableSuffixEnum");
|
|
21
21
|
const GtfsContainerToken_1 = require("../../../ioc/GtfsContainerToken");
|
|
22
22
|
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
23
|
+
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
23
24
|
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
|
|
24
25
|
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
|
|
25
26
|
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
26
27
|
const constants_1 = require("../constants");
|
|
27
28
|
const PrecomputedTablesFacade_1 = require("./helpers/PrecomputedTablesFacade");
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
const PrecomputedTablesLock_1 = require("./helpers/PrecomputedTablesLock");
|
|
30
|
+
const RefreshPrecomputedTablesSchema_1 = require("./schema/RefreshPrecomputedTablesSchema");
|
|
31
|
+
/**
|
|
32
|
+
* Rebuilds the precomputed tables from the live (actual) GTFS tables. They hold a rolling date window
|
|
33
|
+
* (see ServicesCalendarRepository), so every feed needs a rebuild once a day even when its timetable
|
|
34
|
+
* did not change.
|
|
35
|
+
* - without feedCode (daily cron): picks the feeds that have data and were not rebuilt today yet (e.g. by
|
|
36
|
+
* an import) and starts a chain -- one message per feed, each sending the next one when done, so no
|
|
37
|
+
* single message stays unacknowledged for the whole run
|
|
38
|
+
* - with feedCode: rebuilds that feed, then passes the chain on (a failing feed does not stop it); the
|
|
39
|
+
* import failure paths send a lone feedCode, which is a chain of one
|
|
40
|
+
*/
|
|
41
|
+
let RefreshPrecomputedTablesTask = class RefreshPrecomputedTablesTask extends workers_1.AbstractTask {
|
|
42
|
+
constructor(gtfsRepository, precomputeTablesFacade, precomputedTablesLock, gtfsFacade, feedRepository, logger) {
|
|
30
43
|
super(constants_1.TIMETABLE_WORKER_NAME);
|
|
31
44
|
this.gtfsRepository = gtfsRepository;
|
|
32
45
|
this.precomputeTablesFacade = precomputeTablesFacade;
|
|
46
|
+
this.precomputedTablesLock = precomputedTablesLock;
|
|
33
47
|
this.gtfsFacade = gtfsFacade;
|
|
34
48
|
this.feedRepository = feedRepository;
|
|
35
49
|
this.logger = logger;
|
|
36
50
|
this.queueName = "refreshPrecomputedTables";
|
|
37
51
|
this.queueTtl = 59 * 60 * 1000; // 59 minutes
|
|
52
|
+
this.schema = RefreshPrecomputedTablesSchema_1.RefreshPrecomputedTablesSchema;
|
|
38
53
|
this.dataCacheManager = DataCacheManager_1.DataCacheManager.getInstance();
|
|
39
54
|
}
|
|
40
|
-
async execute() {
|
|
41
|
-
|
|
42
|
-
|
|
55
|
+
async execute(data) {
|
|
56
|
+
if (!data.feedCode) {
|
|
57
|
+
await this.startChain();
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const nextFeedCodes = data.nextFeedCodes ?? [];
|
|
61
|
+
let refreshError = null;
|
|
62
|
+
try {
|
|
63
|
+
await this.refreshFeed(data.feedCode);
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
refreshError = err;
|
|
67
|
+
}
|
|
68
|
+
if (nextFeedCodes.length > 0) {
|
|
69
|
+
await this.sendRefresh(nextFeedCodes);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
await this.gtfsRepository.recreateShapesForMvtView();
|
|
73
|
+
}
|
|
74
|
+
if (refreshError instanceof golemio_errors_1.AbstractGolemioError) {
|
|
75
|
+
throw refreshError;
|
|
76
|
+
}
|
|
77
|
+
else if (refreshError) {
|
|
78
|
+
throw new golemio_errors_1.GeneralError(`Error while refreshing Gtfs precomputed tables (feed ${data.feedCode})`, this.constructor.name, refreshError);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async startChain() {
|
|
82
|
+
const feedCodes = [];
|
|
83
|
+
for (const feedCode of await this.feedRepository.getAllFeedCodes()) {
|
|
43
84
|
if (!(await this.gtfsRepository.hasDataForFeed(feedCode))) {
|
|
44
85
|
this.logger.info(`Skipping precomputed refresh for feed ${feedCode} - no data ingested yet`);
|
|
45
|
-
continue;
|
|
46
86
|
}
|
|
47
|
-
await this.
|
|
87
|
+
else if (await this.precomputeTablesFacade.isUpToDate(feedCode)) {
|
|
88
|
+
this.logger.info(`Skipping precomputed refresh for feed ${feedCode} - already rebuilt today`);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
feedCodes.push(feedCode);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (feedCodes.length === 0) {
|
|
95
|
+
this.logger.info("Precomputed tables of all feeds are up to date");
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
this.logger.info(`Starting precomputed refresh chain for feeds: ${feedCodes.join(", ")}`);
|
|
99
|
+
await this.sendRefresh(feedCodes);
|
|
100
|
+
}
|
|
101
|
+
async sendRefresh([feedCode, ...nextFeedCodes]) {
|
|
102
|
+
const message = { feedCode, nextFeedCodes };
|
|
103
|
+
await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, this.queueName, message);
|
|
104
|
+
}
|
|
105
|
+
async refreshFeed(feedCode) {
|
|
106
|
+
if (!(await this.gtfsRepository.hasDataForFeed(feedCode))) {
|
|
107
|
+
this.logger.info(`Skipping precomputed refresh for feed ${feedCode} - no data ingested yet`);
|
|
108
|
+
return;
|
|
48
109
|
}
|
|
110
|
+
await this.precomputedTablesLock.runExclusive(async () => {
|
|
111
|
+
const startedAt = Date.now();
|
|
112
|
+
await this.rebuildFeed(feedCode);
|
|
113
|
+
this.logger.info(`Precomputed tables of feed ${feedCode} refreshed in ${Date.now() - startedAt} ms`);
|
|
114
|
+
});
|
|
49
115
|
await this.dataCacheManager.cleanCache();
|
|
50
|
-
await this.gtfsRepository.recreateShapesForMvtView();
|
|
51
116
|
}
|
|
52
|
-
async
|
|
117
|
+
async rebuildFeed(feedCode) {
|
|
53
118
|
try {
|
|
54
119
|
await this.precomputeTablesFacade.createAndPopulatePrecomputedTmpTables(SourceTableSuffixEnum_1.SourceTableSuffixEnum.Actual, feedCode);
|
|
55
120
|
}
|
|
56
121
|
catch (err) {
|
|
57
|
-
this.
|
|
58
|
-
throw new golemio_errors_1.GeneralError(`Error while creating Gtfs precomputed tables (feed ${feedCode})`, this.constructor.name);
|
|
122
|
+
throw new golemio_errors_1.GeneralError(`Error while creating Gtfs precomputed tables (feed ${feedCode})`, this.constructor.name, err);
|
|
59
123
|
}
|
|
60
124
|
try {
|
|
61
125
|
const feedId = await this.feedRepository.getFeedIdByCode(feedCode);
|
|
62
126
|
await this.gtfsFacade.replacePrecomputedTables(feedCode, feedId);
|
|
63
127
|
}
|
|
64
128
|
catch (err) {
|
|
65
|
-
this.
|
|
66
|
-
throw new golemio_errors_1.GeneralError(`Error while replacing Gtfs precomputed tables (feed ${feedCode})`, this.constructor.name);
|
|
129
|
+
throw new golemio_errors_1.GeneralError(`Error while replacing Gtfs precomputed tables (feed ${feedCode})`, this.constructor.name, err);
|
|
67
130
|
}
|
|
68
131
|
finally {
|
|
69
132
|
await this.gtfsRepository.cleanOldTables(feedCode);
|
|
@@ -75,11 +138,13 @@ exports.RefreshPrecomputedTablesTask = RefreshPrecomputedTablesTask = __decorate
|
|
|
75
138
|
(0, tsyringe_1.injectable)(),
|
|
76
139
|
__param(0, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsRepository)),
|
|
77
140
|
__param(1, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.PrecomputedTablesFacade)),
|
|
78
|
-
__param(2, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.
|
|
79
|
-
__param(3, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.
|
|
80
|
-
__param(4, (0, tsyringe_1.inject)(
|
|
141
|
+
__param(2, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.PrecomputedTablesLock)),
|
|
142
|
+
__param(3, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsFacade)),
|
|
143
|
+
__param(4, (0, tsyringe_1.inject)(GtfsContainerToken_1.GtfsContainerToken.GtfsFeedRepository)),
|
|
144
|
+
__param(5, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
|
|
81
145
|
__metadata("design:paramtypes", [GtfsRepository_1.GtfsRepository,
|
|
82
146
|
PrecomputedTablesFacade_1.PrecomputedTablesFacade,
|
|
147
|
+
PrecomputedTablesLock_1.PrecomputedTablesLock,
|
|
83
148
|
GtfsFacade_1.GtfsFacade,
|
|
84
149
|
GtfsFeedRepository_1.GtfsFeedRepository, Object])
|
|
85
150
|
], RefreshPrecomputedTablesTask);
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RefreshPrecomputedTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,oDAAiD;AACjD,gFAA6E;AAC7E,wEAAqE;AACrE,wEAAqE;AACrE,kFAA+E;AAC/E,wEAAqE;AACrE,wEAAqE;AAErE,
|
|
1
|
+
{"version":3,"file":"RefreshPrecomputedTablesTask.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/RefreshPrecomputedTablesTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,oDAAiD;AACjD,gFAA6E;AAC7E,wEAAqE;AACrE,wEAAqE;AACrE,kFAA+E;AAC/E,wEAAqE;AACrE,wEAAqE;AAErE,8EAAqE;AACrE,2EAA6E;AAC7E,6EAA8F;AAC9F,iEAAwE;AACxE,4CAAqD;AACrD,+EAA4E;AAC5E,2EAAwE;AAExE,4FAAyF;AAEzF;;;;;;;;;GASG;AAEI,IAAM,4BAA4B,GAAlC,MAAM,4BAA6B,SAAQ,sBAA4C;IAO1F,YAC+C,cAA+C,EACtC,sBAAgE,EAClE,qBAA6D,EACxE,UAAuC,EAC/B,cAAmD,EACxE,MAAgC;QAE1D,KAAK,CAAC,iCAAqB,CAAC,CAAC;QAP+B,mBAAc,GAAd,cAAc,CAAgB;QACrB,2BAAsB,GAAtB,sBAAsB,CAAyB;QACjD,0BAAqB,GAArB,qBAAqB,CAAuB;QACvD,eAAU,GAAV,UAAU,CAAY;QACd,mBAAc,GAAd,cAAc,CAAoB;QACvD,WAAM,GAAN,MAAM,CAAS;QAZ9C,cAAS,GAAG,0BAA0B,CAAC;QACvC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;QACxC,WAAM,GAAG,+DAA8B,CAAC;QAEvC,qBAAgB,GAAqB,mCAAgB,CAAC,WAAW,EAAE,CAAC;IAWrF,CAAC;IAES,KAAK,CAAC,OAAO,CAAC,IAAoC;QACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxB,OAAO;QACX,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC;QAC/C,IAAI,YAAY,GAAiB,IAAI,CAAC;QACtC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,YAAY,GAAG,GAAY,CAAC;QAChC,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC1C,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC;QACzD,CAAC;QAED,IAAI,YAAY,YAAY,qCAAoB,EAAE,CAAC;YAC/C,MAAM,YAAY,CAAC;QACvB,CAAC;aAAM,IAAI,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,6BAAY,CAClB,wDAAwD,IAAI,CAAC,QAAQ,GAAG,EACxE,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,YAAY,CACf,CAAC;QACN,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU;QACpB,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,MAAM,QAAQ,IAAI,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,EAAE,CAAC;YACjE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,QAAQ,yBAAyB,CAAC,CAAC;YACjG,CAAC;iBAAM,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,QAAQ,0BAA0B,CAAC,CAAC;YAClG,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YACnE,OAAO;QACX,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1F,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,aAAa,CAAW;QAC5D,MAAM,OAAO,GAAmC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC;QAC5E,MAAM,iCAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACxF,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAgB;QACtC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,QAAQ,yBAAyB,CAAC,CAAC;YAC7F,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,QAAQ,iBAAiB,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,KAAK,CAAC,CAAC;QACzG,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAgB;QACtC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,6CAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,6BAAY,CAAC,sDAAsD,QAAQ,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1H,CAAC;QAED,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACnE,MAAM,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,6BAAY,CAClB,uDAAuD,QAAQ,GAAG,EAClE,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CAAC;QACN,CAAC;gBAAS,CAAC;YACP,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;CACJ,CAAA;AA7GY,oEAA4B;uCAA5B,4BAA4B;IADxC,IAAA,qBAAU,GAAE;IASJ,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,cAAc,CAAC,CAAA;IACzC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,uBAAuB,CAAC,CAAA;IAClD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,qBAAqB,CAAC,CAAA;IAChD,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,UAAU,CAAC,CAAA;IACrC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,kBAAkB,CAAC,CAAA;IAC7C,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;qCALmD,+BAAc;QACG,iDAAuB;QAC1B,6CAAqB;QAC3C,uBAAU;QACE,uCAAkB;GAZ7F,4BAA4B,CA6GxC"}
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.d.ts
CHANGED
|
@@ -14,5 +14,9 @@ export declare class PrecomputedTablesFacade {
|
|
|
14
14
|
private tripConnectionsRepository;
|
|
15
15
|
private servicesCalendarRepository;
|
|
16
16
|
constructor(logger: ILogger, gtfsRepository: GtfsRepository, departureRepository: DeparturesRepository, feedRepository: GtfsFeedRepository);
|
|
17
|
+
/**
|
|
18
|
+
* Tells whether the feed's live precomputed tables were already built today, so a daily refresh can skip it.
|
|
19
|
+
*/
|
|
20
|
+
isUpToDate(feedCode: string): Promise<boolean>;
|
|
17
21
|
createAndPopulatePrecomputedTmpTables: (sourceTableSuffix: SourceTableSuffixEnum, feedCode: string) => Promise<void>;
|
|
18
22
|
}
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.js
CHANGED
|
@@ -51,6 +51,12 @@ let PrecomputedTablesFacade = class PrecomputedTablesFacade {
|
|
|
51
51
|
this.tripScheduleRepository = new TripScheduleRepository_1.TripScheduleRepository();
|
|
52
52
|
this.servicesCalendarRepository = new ServicesCalendarRepository_1.ServicesCalendarRepository();
|
|
53
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Tells whether the feed's live precomputed tables were already built today, so a daily refresh can skip it.
|
|
56
|
+
*/
|
|
57
|
+
async isUpToDate(feedCode) {
|
|
58
|
+
return this.servicesCalendarRepository.hasCurrentWindow(feedCode);
|
|
59
|
+
}
|
|
54
60
|
};
|
|
55
61
|
exports.PrecomputedTablesFacade = PrecomputedTablesFacade;
|
|
56
62
|
exports.PrecomputedTablesFacade = PrecomputedTablesFacade = __decorate([
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PrecomputedTablesFacade.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mFAA6E;AAC7E,2EAAqE;AACrE,mGAA6F;AAC7F,qHAA+G;AAC/G,6GAAuG;AACvG,+GAAyG;AACzG,uGAAiG;AAEjG,2EAAqE;AACrE,wEAAqE;AAErE,iEAAwE;AAGjE,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAOhC,YAC8B,MAAgC,EACf,cAA+C,EAC1C,mBAA0D,EAC3D,cAAmD;QAHvD,WAAM,GAAN,MAAM,CAAS;QACE,mBAAc,GAAd,cAAc,CAAgB;QACzB,wBAAmB,GAAnB,mBAAmB,CAAsB;QAC1C,mBAAc,GAAd,cAAc,CAAoB;
|
|
1
|
+
{"version":3,"file":"PrecomputedTablesFacade.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesFacade.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mFAA6E;AAC7E,2EAAqE;AACrE,mGAA6F;AAC7F,qHAA+G;AAC/G,6GAAuG;AACvG,+GAAyG;AACzG,uGAAiG;AAEjG,2EAAqE;AACrE,wEAAqE;AAErE,iEAAwE;AAGjE,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAOhC,YAC8B,MAAgC,EACf,cAA+C,EAC1C,mBAA0D,EAC3D,cAAmD;QAHvD,WAAM,GAAN,MAAM,CAAS;QACE,mBAAc,GAAd,cAAc,CAAgB;QACzB,wBAAmB,GAAnB,mBAAmB,CAAsB;QAC1C,mBAAc,GAAd,cAAc,CAAoB;QAgB/F,0CAAqC,GAAG,KAAK,EAChD,iBAAwC,EACxC,QAAgB,EACH,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;YAE/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAEnE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,0BAA0B,CAAC,WAAW,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;YACvH,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEpF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,6BAA6B,CAAC,WAAW,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;YAC1H,MAAM,IAAI,CAAC,6BAA6B,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEvF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;YACnH,MAAM,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEhF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;YACtH,MAAM,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEnF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;YAChH,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEtF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;YAC7G,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC,CAAC;QAvCE,oCAAoC;QACpC,IAAI,CAAC,6BAA6B,GAAG,IAAI,6DAA6B,EAAE,CAAC;QACzE,IAAI,CAAC,yBAAyB,GAAG,IAAI,qDAAyB,EAAE,CAAC;QACjE,IAAI,CAAC,sBAAsB,GAAG,IAAI,+CAAsB,EAAE,CAAC;QAC3D,IAAI,CAAC,0BAA0B,GAAG,IAAI,uDAA0B,EAAE,CAAC;IACvE,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,UAAU,CAAC,QAAgB;QACpC,OAAO,IAAI,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;CA4BJ,CAAA;AArDY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,qBAAU,GAAE;IASJ,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;IACxB,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,cAAc,CAAC,CAAA;IACzC,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,mBAAmB,CAAC,CAAA;IAC9C,WAAA,IAAA,iBAAM,EAAC,uCAAkB,CAAC,kBAAkB,CAAC,CAAA;6CAF8B,+BAAc;QACJ,2CAAoB;QAC1B,uCAAkB;GAX7F,uBAAuB,CAqDnC"}
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { IoRedisConnector } from "@golemio/core/dist/helpers/data-access/redis/IoRedisConnector";
|
|
2
|
+
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
3
|
+
/**
|
|
4
|
+
* Serializes every precomputed-table rebuild across all feeds and worker replicas. A rebuild is heavy on
|
|
5
|
+
* the database (full scans of the feed's stop times, a large insert, ANALYZE), so only one runs at a time;
|
|
6
|
+
* it also keeps an import and a refresh of the same feed from dropping each other's _tmp_ tables.
|
|
7
|
+
*/
|
|
8
|
+
export declare class PrecomputedTablesLock {
|
|
9
|
+
private readonly redisClient;
|
|
10
|
+
private readonly logger;
|
|
11
|
+
constructor(redisClient: IoRedisConnector, logger: ILogger);
|
|
12
|
+
/**
|
|
13
|
+
* Waits for the lock, runs the action while holding it and releases it afterwards.
|
|
14
|
+
*/
|
|
15
|
+
runExclusive<T>(action: () => Promise<T>): Promise<T>;
|
|
16
|
+
}
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.js
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.PrecomputedTablesLock = void 0;
|
|
16
|
+
const IoRedisConnector_1 = require("@golemio/core/dist/helpers/data-access/redis/IoRedisConnector");
|
|
17
|
+
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
18
|
+
const ioc_1 = require("@golemio/core/dist/integration-engine/ioc");
|
|
19
|
+
const redis_semaphore_1 = require("@golemio/core/dist/shared/redis-semaphore");
|
|
20
|
+
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
21
|
+
// pt-prefixed so it never collides with @golemio/pid's lock keys on the shared Redis
|
|
22
|
+
const LOCK_KEY = "ptGtfsPrecomputedTables";
|
|
23
|
+
// kept alive by the mutex's auto-refresh while the holder runs, expires quickly if the holder dies
|
|
24
|
+
const LOCK_TIMEOUT_MS = 60 * 1000;
|
|
25
|
+
const LOCK_REFRESH_INTERVAL_MS = 20 * 1000;
|
|
26
|
+
// one feed's rebuild takes minutes at most; a waiter gives up well before RabbitMQ's consumer timeout
|
|
27
|
+
const ACQUIRE_TIMEOUT_MS = 20 * 60 * 1000;
|
|
28
|
+
const ACQUIRE_RETRY_INTERVAL_MS = 1000;
|
|
29
|
+
/**
|
|
30
|
+
* Serializes every precomputed-table rebuild across all feeds and worker replicas. A rebuild is heavy on
|
|
31
|
+
* the database (full scans of the feed's stop times, a large insert, ANALYZE), so only one runs at a time;
|
|
32
|
+
* it also keeps an import and a refresh of the same feed from dropping each other's _tmp_ tables.
|
|
33
|
+
*/
|
|
34
|
+
let PrecomputedTablesLock = class PrecomputedTablesLock {
|
|
35
|
+
constructor(redisClient, logger) {
|
|
36
|
+
this.redisClient = redisClient;
|
|
37
|
+
this.logger = logger;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Waits for the lock, runs the action while holding it and releases it afterwards.
|
|
41
|
+
*/
|
|
42
|
+
async runExclusive(action) {
|
|
43
|
+
const mutex = new redis_semaphore_1.Mutex(this.redisClient.getConnection(), LOCK_KEY, {
|
|
44
|
+
lockTimeout: LOCK_TIMEOUT_MS,
|
|
45
|
+
refreshInterval: LOCK_REFRESH_INTERVAL_MS,
|
|
46
|
+
acquireTimeout: ACQUIRE_TIMEOUT_MS,
|
|
47
|
+
retryInterval: ACQUIRE_RETRY_INTERVAL_MS,
|
|
48
|
+
onLockLost: (err) => {
|
|
49
|
+
this.logger.error(err, `${this.constructor.name}: mutex lock was lost`);
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
await mutex.acquire();
|
|
53
|
+
try {
|
|
54
|
+
return await action();
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
await mutex.release();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
exports.PrecomputedTablesLock = PrecomputedTablesLock;
|
|
62
|
+
exports.PrecomputedTablesLock = PrecomputedTablesLock = __decorate([
|
|
63
|
+
(0, tsyringe_1.injectable)(),
|
|
64
|
+
__param(0, (0, tsyringe_1.inject)(ioc_1.ContainerToken.RedisConnector)),
|
|
65
|
+
__param(1, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
|
|
66
|
+
__metadata("design:paramtypes", [IoRedisConnector_1.IoRedisConnector, Object])
|
|
67
|
+
], PrecomputedTablesLock);
|
|
68
|
+
//# sourceMappingURL=PrecomputedTablesLock.js.map
|
package/dist/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PrecomputedTablesLock.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/helpers/PrecomputedTablesLock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,oGAAiG;AACjG,wEAAqE;AAErE,mEAA2E;AAC3E,+EAAkE;AAClE,iEAAwE;AAExE,qFAAqF;AACrF,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAC3C,mGAAmG;AACnG,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC;AAClC,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,sGAAsG;AACtG,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC1C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AAEvC;;;;GAIG;AAEI,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IAC9B,YAC4D,WAA6B,EAC1C,MAAe;QADF,gBAAW,GAAX,WAAW,CAAkB;QAC1C,WAAM,GAAN,MAAM,CAAS;IAC3D,CAAC;IAEJ;;OAEG;IACI,KAAK,CAAC,YAAY,CAAI,MAAwB;QACjD,MAAM,KAAK,GAAG,IAAI,uBAAK,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE;YAChE,WAAW,EAAE,eAAe;YAC5B,eAAe,EAAE,wBAAwB;YACzC,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,yBAAyB;YACxC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,uBAAuB,CAAC,CAAC;YAC5E,CAAC;SACJ,CAAC,CAAC;QAEH,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACtB,IAAI,CAAC;YACD,OAAO,MAAM,MAAM,EAAE,CAAC;QAC1B,CAAC;gBAAS,CAAC;YACP,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC;CACJ,CAAA;AA3BY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,qBAAU,GAAE;IAGJ,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,cAAc,CAAC,CAAA;IACrC,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;qCAD4C,mCAAgB;GAFhF,qBAAqB,CA2BjC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IRefreshPrecomputedTablesInput.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/interfaces/IRefreshPrecomputedTablesInput.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.RefreshPrecomputedTablesSchema = void 0;
|
|
13
|
+
const class_validator_1 = require("@golemio/core/dist/shared/class-validator");
|
|
14
|
+
const FEED_CODE_PATTERN = /^[A-Za-z0-9_-]+$/;
|
|
15
|
+
// feed codes are folded into raw-SQL identifiers (feed-suffixed table names), so reject anything outside
|
|
16
|
+
// this charset at the message boundary.
|
|
17
|
+
class RefreshPrecomputedTablesSchema {
|
|
18
|
+
}
|
|
19
|
+
exports.RefreshPrecomputedTablesSchema = RefreshPrecomputedTablesSchema;
|
|
20
|
+
__decorate([
|
|
21
|
+
(0, class_validator_1.IsOptional)(),
|
|
22
|
+
(0, class_validator_1.IsString)(),
|
|
23
|
+
(0, class_validator_1.Matches)(FEED_CODE_PATTERN),
|
|
24
|
+
__metadata("design:type", String)
|
|
25
|
+
], RefreshPrecomputedTablesSchema.prototype, "feedCode", void 0);
|
|
26
|
+
__decorate([
|
|
27
|
+
(0, class_validator_1.IsOptional)(),
|
|
28
|
+
(0, class_validator_1.IsArray)(),
|
|
29
|
+
(0, class_validator_1.IsString)({ each: true }),
|
|
30
|
+
(0, class_validator_1.Matches)(FEED_CODE_PATTERN, { each: true }),
|
|
31
|
+
__metadata("design:type", Array)
|
|
32
|
+
], RefreshPrecomputedTablesSchema.prototype, "nextFeedCodes", void 0);
|
|
33
|
+
//# sourceMappingURL=RefreshPrecomputedTablesSchema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RefreshPrecomputedTablesSchema.js","sourceRoot":"","sources":["../../../../../../../src/integration-engine/gtfs/workers/timetables/tasks/schema/RefreshPrecomputedTablesSchema.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+EAAmG;AAGnG,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AAE7C,yGAAyG;AACzG,wCAAwC;AACxC,MAAa,8BAA8B;CAW1C;AAXD,wEAWC;AAPG;IAHC,IAAA,4BAAU,GAAE;IACZ,IAAA,0BAAQ,GAAE;IACV,IAAA,yBAAO,EAAC,iBAAiB,CAAC;;gEACT;AAMlB;IAJC,IAAA,4BAAU,GAAE;IACZ,IAAA,yBAAO,GAAE;IACT,IAAA,0BAAQ,EAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxB,IAAA,yBAAO,EAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;;qEAClB"}
|
|
@@ -580,11 +580,21 @@ Všechny tabulky se nachází ve schématu `pid`
|
|
|
580
580
|
- vstupní rabbitmq fronta
|
|
581
581
|
- název: vehicle-positions.ropidgtfs.refreshPrecomputedTables
|
|
582
582
|
- TTL: 59 minut
|
|
583
|
-
- parametry:
|
|
583
|
+
- parametry: `{ "feedCode"?: string, "nextFeedCodes"?: string[] }`
|
|
584
|
+
- bez `feedCode` (denní cron, např. ve 3:xx Europe/Prague): vybere feedy s daty, jejichž precomputed tabulky
|
|
585
|
+
ještě dnes nevznikly (např. importem), a spustí řetěz: jedna zpráva na feed, po dokončení feedu se pošle
|
|
586
|
+
zpráva pro další (`{ feedCode, nextFeedCodes }`), takže žádná zpráva nezůstane nepotvrzená po celý běh
|
|
587
|
+
- s `feedCode`: obnoví daný feed (vždy) a předá řetěz dál i při chybě; konec řetězu obnoví pohled
|
|
588
|
+
`v_ropidgtfs_shapes_for_mvt`; chybové větve importu posílají samotný `{ feedCode }` (řetěz o jednom feedu)
|
|
589
|
+
- ruční obnova jednoho feedu: publikovat `{ "feedCode": "<kód>" }` do fronty (např. po chybě feedu v řetězu;
|
|
590
|
+
zpráva neúspěšného feedu se neopakuje, jde do dead-letter)
|
|
584
591
|
- závislé fronty (do kterých jsou odesílány zprávy z metody workeru)
|
|
585
|
-
-
|
|
592
|
+
- název: vehicle-positions.ropidgtfs.refreshPrecomputedTables (další feed v řetězu)
|
|
586
593
|
- datové zdroje
|
|
587
|
-
-
|
|
594
|
+
- aktuální (actual) GTFS tabulky feedu
|
|
595
|
+
- souběh: přestavba precomputed tabulek je náročná na DB, proto ji (včetně importu v `CheckSavedRowsAndReplaceTables`)
|
|
596
|
+
serializuje Redis mutex `ptGtfsPrecomputedTables` napříč feedy i replikami workeru
|
|
597
|
+
- precomputed tabulky drží klouzavé okno dat (včera až +2 dny), proto musí být každý feed obnoven alespoň jednou denně
|
|
588
598
|
- transformace
|
|
589
599
|
- žádné
|
|
590
600
|
- data modely
|