@golemio/chmu 1.1.1-dev.2647709694 → 1.1.1-dev.2674913750
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/integration-engine/datasources/AbstractOpendataDatasource.d.ts +19 -0
- package/dist/integration-engine/datasources/AbstractOpendataDatasource.js +47 -0
- package/dist/integration-engine/datasources/AbstractOpendataDatasource.js.map +1 -0
- package/dist/integration-engine/datasources/OpendataDatasource.d.ts +3 -25
- package/dist/integration-engine/datasources/OpendataDatasource.js +3 -46
- package/dist/integration-engine/datasources/OpendataDatasource.js.map +1 -1
- package/dist/integration-engine/datasources/OpendataHistoricalDatasource.d.ts +8 -0
- package/dist/integration-engine/datasources/OpendataHistoricalDatasource.js +38 -0
- package/dist/integration-engine/datasources/OpendataHistoricalDatasource.js.map +1 -0
- package/dist/integration-engine/ioc/Di.js +4 -0
- package/dist/integration-engine/ioc/Di.js.map +1 -1
- package/dist/integration-engine/ioc/ModuleContainerToken.d.ts +2 -0
- package/dist/integration-engine/ioc/ModuleContainerToken.js +3 -0
- package/dist/integration-engine/ioc/ModuleContainerToken.js.map +1 -1
- package/dist/integration-engine/workers/ChmuWorker.js +1 -0
- package/dist/integration-engine/workers/ChmuWorker.js.map +1 -1
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.d.ts +10 -0
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.js +3 -0
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.js.map +1 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.d.ts +26 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.js +149 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.js.map +1 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.d.ts +7 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.js +40 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.js.map +1 -0
- package/dist/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.d.ts +1 -1
- package/dist/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.js +5 -5
- package/dist/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.js.map +1 -1
- package/docs/asyncapi.yaml +44 -1
- package/docs/implementation_documentation.md +33 -0
- package/package.json +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Raw record shape shared by both the live (daily) and historical (monthly) ČHMÚ opendata sources. */
|
|
2
|
+
export interface IRawOpendataRecord {
|
|
3
|
+
STATION: string;
|
|
4
|
+
ELEMENT: string;
|
|
5
|
+
DT: string;
|
|
6
|
+
VAL: number | "";
|
|
7
|
+
FLAG: string;
|
|
8
|
+
QUALITY: number;
|
|
9
|
+
}
|
|
10
|
+
export declare abstract class AbstractOpendataDatasource {
|
|
11
|
+
abstract name: string;
|
|
12
|
+
private readonly responseValidator;
|
|
13
|
+
private readonly recordValidator;
|
|
14
|
+
protected abstract buildUrl(wsi: string, date: Date): string;
|
|
15
|
+
getData(wsi: string, date?: Date): Promise<IRawOpendataRecord[]>;
|
|
16
|
+
private getDatasource;
|
|
17
|
+
private parseHeader;
|
|
18
|
+
private mapToRecords;
|
|
19
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.AbstractOpendataDatasource = void 0;
|
|
7
|
+
const datasources_1 = require("@golemio/core/dist/integration-engine/datasources");
|
|
8
|
+
const HTTPFetchProtocolStrategy_1 = require("@golemio/core/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy");
|
|
9
|
+
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
10
|
+
const OpendataSchemaProvider_1 = __importDefault(require("../../schema-definitions/datasources/OpendataSchemaProvider"));
|
|
11
|
+
const REQUIRED_COLUMNS = ["STATION", "ELEMENT", "DT", "VAL", "FLAG", "QUALITY"];
|
|
12
|
+
class AbstractOpendataDatasource {
|
|
13
|
+
constructor() {
|
|
14
|
+
this.responseValidator = new golemio_validator_1.JSONSchemaValidator("OpendataResponseValidator", OpendataSchemaProvider_1.default.getResponseSchema());
|
|
15
|
+
this.recordValidator = new golemio_validator_1.JSONSchemaValidator("OpendataRecordValidator", OpendataSchemaProvider_1.default.get());
|
|
16
|
+
}
|
|
17
|
+
async getData(wsi, date = new Date()) {
|
|
18
|
+
const { header, values } = await this.getDatasource(this.buildUrl(wsi, date)).getAll();
|
|
19
|
+
const colIndex = this.parseHeader(header);
|
|
20
|
+
const records = this.mapToRecords(values, colIndex);
|
|
21
|
+
await this.recordValidator.Validate(records);
|
|
22
|
+
return records;
|
|
23
|
+
}
|
|
24
|
+
getDatasource(url) {
|
|
25
|
+
return new datasources_1.DataSource("OpendataHttpDataSource", new HTTPFetchProtocolStrategy_1.HTTPFetchProtocolStrategy({ url, method: "GET", responseType: "json" }), new datasources_1.JSONDataTypeStrategy({ resultsPath: "data.data" }), this.responseValidator);
|
|
26
|
+
}
|
|
27
|
+
parseHeader(header) {
|
|
28
|
+
const colIndex = Object.fromEntries(header.split(",").map((c, i) => [c, i]));
|
|
29
|
+
const missing = REQUIRED_COLUMNS.filter((c) => colIndex[c] === undefined);
|
|
30
|
+
if (missing.length > 0) {
|
|
31
|
+
throw new Error(`${this.constructor.name}: missing columns in header: ${missing.join(", ")}`);
|
|
32
|
+
}
|
|
33
|
+
return colIndex;
|
|
34
|
+
}
|
|
35
|
+
mapToRecords(values, colIndex) {
|
|
36
|
+
return values.map((row) => ({
|
|
37
|
+
STATION: row[colIndex["STATION"]],
|
|
38
|
+
ELEMENT: row[colIndex["ELEMENT"]],
|
|
39
|
+
DT: row[colIndex["DT"]],
|
|
40
|
+
VAL: row[colIndex["VAL"]],
|
|
41
|
+
FLAG: row[colIndex["FLAG"]],
|
|
42
|
+
QUALITY: row[colIndex["QUALITY"]],
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
exports.AbstractOpendataDatasource = AbstractOpendataDatasource;
|
|
47
|
+
//# sourceMappingURL=AbstractOpendataDatasource.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AbstractOpendataDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/AbstractOpendataDatasource.ts"],"names":[],"mappings":";;;;;;AAAA,mFAAqG;AACrG,6IAA0I;AAC1I,mFAAkF;AAClF,yHAA6E;AAY7E,MAAM,gBAAgB,GAA4C,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzH,MAAsB,0BAA0B;IAAhD;QAGqB,sBAAiB,GAAG,IAAI,uCAAmB,CACxD,2BAA2B,EAC3B,gCAAsB,CAAC,iBAAiB,EAAE,CAC7C,CAAC;QACe,oBAAe,GAAG,IAAI,uCAAmB,CAAC,yBAAyB,EAAE,gCAAsB,CAAC,GAAG,EAAE,CAAC,CAAC;IAwCxH,CAAC;IApCU,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,OAAa,IAAI,IAAI,EAAE;QACrD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC7C,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,aAAa,CAAC,GAAW;QAC7B,OAAO,IAAI,wBAAU,CACjB,wBAAwB,EACxB,IAAI,qDAAyB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,EAC3E,IAAI,kCAAoB,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,EACtD,IAAI,CAAC,iBAAiB,CACzB,CAAC;IACN,CAAC;IAEO,WAAW,CAAC,MAAc;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,YAAY,CAAC,MAAmB,EAAE,QAAgC;QACtE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW;YAC3C,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW;YAC3C,EAAE,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAW;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAgB;YACxC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAW;YACrC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW;SAC9C,CAAC,CAAC,CAAC;IACR,CAAC;CACJ;AA/CD,gEA+CC"}
|
|
@@ -1,30 +1,8 @@
|
|
|
1
1
|
import { ISimpleConfig } from "@golemio/core/dist/helpers/configuration/ISimpleConfig";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
STATION: string;
|
|
5
|
-
ELEMENT: string;
|
|
6
|
-
DT: string;
|
|
7
|
-
VAL: number | "";
|
|
8
|
-
FLAG: string;
|
|
9
|
-
QUALITY: number;
|
|
10
|
-
}
|
|
11
|
-
export default class OpendataDatasource {
|
|
2
|
+
import { AbstractOpendataDatasource } from "./AbstractOpendataDatasource";
|
|
3
|
+
export default class OpendataDatasource extends AbstractOpendataDatasource {
|
|
12
4
|
name: string;
|
|
13
|
-
private readonly responseValidator;
|
|
14
|
-
private readonly recordValidator;
|
|
15
5
|
private readonly baseUrl;
|
|
16
6
|
constructor(config: ISimpleConfig);
|
|
17
|
-
|
|
18
|
-
* Fetches the 10-minute JSON file for one station from ČHMÚ opendata.
|
|
19
|
-
* Returns unpacked records with named fields (STATION = WSI).
|
|
20
|
-
*
|
|
21
|
-
* @param wsi Station WSI identifier (e.g. "0-20000-0-11520")
|
|
22
|
-
* @param date Date for which to fetch data (defaults to today)
|
|
23
|
-
*/
|
|
24
|
-
getData(wsi: string, date?: Date): Promise<IRawOpendataRecord[]>;
|
|
25
|
-
private getDatasource;
|
|
26
|
-
private parseHeader;
|
|
27
|
-
private mapToRecords;
|
|
28
|
-
private buildUrl;
|
|
7
|
+
protected buildUrl(wsi: string, date: Date): string;
|
|
29
8
|
}
|
|
30
|
-
export {};
|
|
@@ -11,60 +11,17 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
11
11
|
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
12
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
13
|
};
|
|
14
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
15
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
16
|
-
};
|
|
17
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
15
|
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
19
|
-
const datasources_1 = require("@golemio/core/dist/integration-engine/datasources");
|
|
20
|
-
const HTTPFetchProtocolStrategy_1 = require("@golemio/core/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy");
|
|
21
16
|
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
22
|
-
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
23
17
|
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
let OpendataDatasource = class OpendataDatasource {
|
|
18
|
+
const AbstractOpendataDatasource_1 = require("./AbstractOpendataDatasource");
|
|
19
|
+
let OpendataDatasource = class OpendataDatasource extends AbstractOpendataDatasource_1.AbstractOpendataDatasource {
|
|
27
20
|
constructor(config) {
|
|
21
|
+
super();
|
|
28
22
|
this.name = "OpendataDatasource";
|
|
29
|
-
this.responseValidator = new golemio_validator_1.JSONSchemaValidator("OpendataResponseValidator", OpendataSchemaProvider_1.default.getResponseSchema());
|
|
30
|
-
this.recordValidator = new golemio_validator_1.JSONSchemaValidator("OpendataRecordValidator", OpendataSchemaProvider_1.default.get());
|
|
31
23
|
this.baseUrl = config.getValue("module.chmu.opendata.baseUrl");
|
|
32
24
|
}
|
|
33
|
-
/**
|
|
34
|
-
* Fetches the 10-minute JSON file for one station from ČHMÚ opendata.
|
|
35
|
-
* Returns unpacked records with named fields (STATION = WSI).
|
|
36
|
-
*
|
|
37
|
-
* @param wsi Station WSI identifier (e.g. "0-20000-0-11520")
|
|
38
|
-
* @param date Date for which to fetch data (defaults to today)
|
|
39
|
-
*/
|
|
40
|
-
async getData(wsi, date = new Date()) {
|
|
41
|
-
const { header, values } = await this.getDatasource(this.buildUrl(wsi, date)).getAll();
|
|
42
|
-
const colIndex = this.parseHeader(header);
|
|
43
|
-
const records = this.mapToRecords(values, colIndex);
|
|
44
|
-
await this.recordValidator.Validate(records);
|
|
45
|
-
return records;
|
|
46
|
-
}
|
|
47
|
-
getDatasource(url) {
|
|
48
|
-
return new datasources_1.DataSource("OpendataHttpDataSource", new HTTPFetchProtocolStrategy_1.HTTPFetchProtocolStrategy({ url, method: "GET", responseType: "json" }), new datasources_1.JSONDataTypeStrategy({ resultsPath: "data.data" }), this.responseValidator);
|
|
49
|
-
}
|
|
50
|
-
parseHeader(header) {
|
|
51
|
-
const colIndex = Object.fromEntries(header.split(",").map((c, i) => [c, i]));
|
|
52
|
-
const missing = REQUIRED_COLUMNS.filter((c) => colIndex[c] === undefined);
|
|
53
|
-
if (missing.length > 0) {
|
|
54
|
-
throw new Error(`OpendataDatasource: missing columns in header: ${missing.join(", ")}`);
|
|
55
|
-
}
|
|
56
|
-
return colIndex;
|
|
57
|
-
}
|
|
58
|
-
mapToRecords(values, colIndex) {
|
|
59
|
-
return values.map((row) => ({
|
|
60
|
-
STATION: row[colIndex["STATION"]],
|
|
61
|
-
ELEMENT: row[colIndex["ELEMENT"]],
|
|
62
|
-
DT: row[colIndex["DT"]],
|
|
63
|
-
VAL: row[colIndex["VAL"]],
|
|
64
|
-
FLAG: row[colIndex["FLAG"]],
|
|
65
|
-
QUALITY: row[colIndex["QUALITY"]],
|
|
66
|
-
}));
|
|
67
|
-
}
|
|
68
25
|
buildUrl(wsi, date) {
|
|
69
26
|
const dt = luxon_1.DateTime.fromJSDate(date).setZone("Europe/Prague");
|
|
70
27
|
const yyyymmdd = dt.toFormat("yyyyMMdd");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpendataDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/OpendataDatasource.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"OpendataDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/OpendataDatasource.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AACA,wEAAqE;AACrE,2DAA2D;AAC3D,iEAAwE;AACxE,6EAA0E;AAG3D,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,uDAA0B;IAKtE,YAA4C,MAAqB;QAC7D,KAAK,EAAE,CAAC;QALL,SAAI,GAAG,oBAAoB,CAAC;QAM/B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC;IACnE,CAAC;IAES,QAAQ,CAAC,GAAW,EAAE,IAAU;QACtC,MAAM,EAAE,GAAG,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACzC,OAAO,GAAG,IAAI,CAAC,OAAO,QAAQ,GAAG,IAAI,QAAQ,OAAO,CAAC;IACzD,CAAC;CACJ,CAAA;AAfoB,kBAAkB;IADtC,IAAA,qBAAU,GAAE;IAMI,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,YAAY,CAAC,CAAA;;GAL1B,kBAAkB,CAetC;kBAfoB,kBAAkB"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { ISimpleConfig } from "@golemio/core/dist/helpers/configuration/ISimpleConfig";
|
|
2
|
+
import { AbstractOpendataDatasource } from "./AbstractOpendataDatasource";
|
|
3
|
+
export default class OpendataHistoricalDatasource extends AbstractOpendataDatasource {
|
|
4
|
+
name: string;
|
|
5
|
+
private readonly baseUrl;
|
|
6
|
+
constructor(config: ISimpleConfig);
|
|
7
|
+
protected buildUrl(wsi: string, date: Date): string;
|
|
8
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
16
|
+
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
17
|
+
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
18
|
+
const AbstractOpendataDatasource_1 = require("./AbstractOpendataDatasource");
|
|
19
|
+
let OpendataHistoricalDatasource = class OpendataHistoricalDatasource extends AbstractOpendataDatasource_1.AbstractOpendataDatasource {
|
|
20
|
+
constructor(config) {
|
|
21
|
+
super();
|
|
22
|
+
this.name = "OpendataHistoricalDatasource";
|
|
23
|
+
this.baseUrl = config.getValue("module.chmu.opendata.historicalBaseUrl");
|
|
24
|
+
}
|
|
25
|
+
buildUrl(wsi, date) {
|
|
26
|
+
const dt = luxon_1.DateTime.fromJSDate(date, { zone: "utc" });
|
|
27
|
+
const year = dt.toFormat("yyyy");
|
|
28
|
+
const yearMonth = dt.toFormat("yyyyMM");
|
|
29
|
+
return `${this.baseUrl}/${year}/10m-${wsi}-${yearMonth}.json`;
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
OpendataHistoricalDatasource = __decorate([
|
|
33
|
+
(0, tsyringe_1.injectable)(),
|
|
34
|
+
__param(0, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.SimpleConfig)),
|
|
35
|
+
__metadata("design:paramtypes", [Object])
|
|
36
|
+
], OpendataHistoricalDatasource);
|
|
37
|
+
exports.default = OpendataHistoricalDatasource;
|
|
38
|
+
//# sourceMappingURL=OpendataHistoricalDatasource.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"OpendataHistoricalDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/OpendataHistoricalDatasource.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AACA,wEAAqE;AACrE,2DAA2D;AAC3D,iEAAwE;AACxE,6EAA0E;AAG3D,IAAM,4BAA4B,GAAlC,MAAM,4BAA6B,SAAQ,uDAA0B;IAKhF,YAA4C,MAAqB;QAC7D,KAAK,EAAE,CAAC;QALL,SAAI,GAAG,8BAA8B,CAAC;QAMzC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,wCAAwC,CAAC,CAAC;IAC7E,CAAC;IAES,QAAQ,CAAC,GAAW,EAAE,IAAU;QACtC,MAAM,EAAE,GAAG,gBAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,QAAQ,GAAG,IAAI,SAAS,OAAO,CAAC;IAClE,CAAC;CACJ,CAAA;AAhBoB,4BAA4B;IADhD,IAAA,qBAAU,GAAE;IAMI,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,YAAY,CAAC,CAAA;;GAL1B,4BAA4B,CAgBhD;kBAhBoB,4BAA4B"}
|
|
@@ -9,10 +9,12 @@ const StationsRepository_1 = require("../data-access/StationsRepository");
|
|
|
9
9
|
const Di_1 = require("@golemio/core/dist/integration-engine/ioc/Di");
|
|
10
10
|
const ChmuDatasource_1 = __importDefault(require("../datasources/ChmuDatasource"));
|
|
11
11
|
const OpendataDatasource_1 = __importDefault(require("../datasources/OpendataDatasource"));
|
|
12
|
+
const OpendataHistoricalDatasource_1 = __importDefault(require("../datasources/OpendataHistoricalDatasource"));
|
|
12
13
|
const MeasurementsTransformation_1 = require("../transformations/MeasurementsTransformation");
|
|
13
14
|
const OpendataMeasurementsTransformation_1 = require("../transformations/OpendataMeasurementsTransformation");
|
|
14
15
|
const DownloadMeasurementsTask_1 = __importDefault(require("../workers/tasks/DownloadMeasurementsTask"));
|
|
15
16
|
const DownloadOpendataMeasurementsTask_1 = __importDefault(require("../workers/tasks/DownloadOpendataMeasurementsTask"));
|
|
17
|
+
const DownloadOpendataHistoricalMeasurementsTask_1 = __importDefault(require("../workers/tasks/DownloadOpendataHistoricalMeasurementsTask"));
|
|
16
18
|
const ModuleContainerToken_1 = require("./ModuleContainerToken");
|
|
17
19
|
//#region Initialization
|
|
18
20
|
const ChmuContainer = Di_1.IntegrationEngineContainer.createChildContainer();
|
|
@@ -21,6 +23,7 @@ exports.ChmuContainer = ChmuContainer;
|
|
|
21
23
|
//#region Datasources
|
|
22
24
|
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.ChmuDatasource, ChmuDatasource_1.default);
|
|
23
25
|
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.OpendataDatasource, OpendataDatasource_1.default);
|
|
26
|
+
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.OpendataHistoricalDatasource, OpendataHistoricalDatasource_1.default);
|
|
24
27
|
//#endregion
|
|
25
28
|
//#region Transformations
|
|
26
29
|
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.MeasurementsTransformation, MeasurementsTransformation_1.MeasurementsTransformation);
|
|
@@ -33,4 +36,5 @@ ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.Stat
|
|
|
33
36
|
//#region WorkerTasks
|
|
34
37
|
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.DownloadMeasurementsTask, DownloadMeasurementsTask_1.default);
|
|
35
38
|
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.DownloadOpendataMeasurementsTask, DownloadOpendataMeasurementsTask_1.default);
|
|
39
|
+
ChmuContainer.registerSingleton(ModuleContainerToken_1.ModuleContainerToken.DownloadOpendataHistoricalMeasurementsTask, DownloadOpendataHistoricalMeasurementsTask_1.default);
|
|
36
40
|
//# sourceMappingURL=Di.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../src/integration-engine/ioc/Di.ts"],"names":[],"mappings":";;;;;;AAAA,kFAAgF;AAChF,0EAAwE;AACxE,qEAA0F;AAC1F,mFAA4D;AAC5D,2FAAoE;AACpE,8FAA4F;AAC5F,8GAA4G;AAC5G,yGAAkF;AAClF,yHAAkG;
|
|
1
|
+
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../src/integration-engine/ioc/Di.ts"],"names":[],"mappings":";;;;;;AAAA,kFAAgF;AAChF,0EAAwE;AACxE,qEAA0F;AAC1F,mFAA4D;AAC5D,2FAAoE;AACpE,+GAAwF;AACxF,8FAA4F;AAC5F,8GAA4G;AAC5G,yGAAkF;AAClF,yHAAkG;AAClG,6IAAsH;AAEtH,iEAA8D;AAE9D,wBAAwB;AACxB,MAAM,aAAa,GAAwB,+BAA0B,CAAC,oBAAoB,EAAE,CAAC;AA4BpF,sCAAa;AA3BtB,YAAY;AAEZ,qBAAqB;AACrB,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,cAAc,EAAE,wBAAc,CAAC,CAAC;AACrF,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,kBAAkB,EAAE,4BAAkB,CAAC,CAAC;AAC7F,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,4BAA4B,EAAE,sCAA4B,CAAC,CAAC;AACjH,YAAY;AAEZ,yBAAyB;AACzB,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,0BAA0B,EAAE,uDAA0B,CAAC,CAAC;AAC7G,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,kCAAkC,EAAE,uEAAkC,CAAC,CAAC;AAC7H,YAAY;AAEZ,sBAAsB;AACtB,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,sBAAsB,EAAE,+CAAsB,CAAC,CAAC;AACrG,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,kBAAkB,EAAE,uCAAkB,CAAC,CAAC;AAC7F,YAAY;AAEZ,qBAAqB;AACrB,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,wBAAwB,EAAE,kCAAwB,CAAC,CAAC;AACzG,aAAa,CAAC,iBAAiB,CAAC,2CAAoB,CAAC,gCAAgC,EAAE,0CAAgC,CAAC,CAAC;AACzH,aAAa,CAAC,iBAAiB,CAC3B,2CAAoB,CAAC,0CAA0C,EAC/D,oDAA0C,CAC7C,CAAC"}
|
|
@@ -7,5 +7,7 @@ declare const ModuleContainerToken: {
|
|
|
7
7
|
OpendataDatasource: symbol;
|
|
8
8
|
OpendataMeasurementsTransformation: symbol;
|
|
9
9
|
DownloadOpendataMeasurementsTask: symbol;
|
|
10
|
+
OpendataHistoricalDatasource: symbol;
|
|
11
|
+
DownloadOpendataHistoricalMeasurementsTask: symbol;
|
|
10
12
|
};
|
|
11
13
|
export { ModuleContainerToken };
|
|
@@ -11,6 +11,9 @@ const ModuleContainerToken = {
|
|
|
11
11
|
OpendataDatasource: Symbol(),
|
|
12
12
|
OpendataMeasurementsTransformation: Symbol(),
|
|
13
13
|
DownloadOpendataMeasurementsTask: Symbol(),
|
|
14
|
+
// opendata historical
|
|
15
|
+
OpendataHistoricalDatasource: Symbol(),
|
|
16
|
+
DownloadOpendataHistoricalMeasurementsTask: Symbol(),
|
|
14
17
|
};
|
|
15
18
|
exports.ModuleContainerToken = ModuleContainerToken;
|
|
16
19
|
//# sourceMappingURL=ModuleContainerToken.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ModuleContainerToken.js","sourceRoot":"","sources":["../../../src/integration-engine/ioc/ModuleContainerToken.ts"],"names":[],"mappings":";;;AAAA,MAAM,oBAAoB,GAAG;IACzB,cAAc,EAAE,MAAM,EAAE;IACxB,sBAAsB,EAAE,MAAM,EAAE;IAChC,0BAA0B,EAAE,MAAM,EAAE;IACpC,wBAAwB,EAAE,MAAM,EAAE;IAClC,oBAAoB;IACpB,kBAAkB,EAAE,MAAM,EAAE;IAC5B,kBAAkB,EAAE,MAAM,EAAE;IAC5B,kCAAkC,EAAE,MAAM,EAAE;IAC5C,gCAAgC,EAAE,MAAM,EAAE;
|
|
1
|
+
{"version":3,"file":"ModuleContainerToken.js","sourceRoot":"","sources":["../../../src/integration-engine/ioc/ModuleContainerToken.ts"],"names":[],"mappings":";;;AAAA,MAAM,oBAAoB,GAAG;IACzB,cAAc,EAAE,MAAM,EAAE;IACxB,sBAAsB,EAAE,MAAM,EAAE;IAChC,0BAA0B,EAAE,MAAM,EAAE;IACpC,wBAAwB,EAAE,MAAM,EAAE;IAClC,oBAAoB;IACpB,kBAAkB,EAAE,MAAM,EAAE;IAC5B,kBAAkB,EAAE,MAAM,EAAE;IAC5B,kCAAkC,EAAE,MAAM,EAAE;IAC5C,gCAAgC,EAAE,MAAM,EAAE;IAC1C,sBAAsB;IACtB,4BAA4B,EAAE,MAAM,EAAE;IACtC,0CAA0C,EAAE,MAAM,EAAE;CACvD,CAAC;AAEO,oDAAoB"}
|
|
@@ -16,6 +16,7 @@ class ChmuWorker extends workers_1.AbstractWorker {
|
|
|
16
16
|
// Register tasks
|
|
17
17
|
this.registerTask(Di_1.ChmuContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.DownloadMeasurementsTask));
|
|
18
18
|
this.registerTask(Di_1.ChmuContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.DownloadOpendataMeasurementsTask));
|
|
19
|
+
this.registerTask(Di_1.ChmuContainer.resolve(ModuleContainerToken_1.ModuleContainerToken.DownloadOpendataHistoricalMeasurementsTask));
|
|
19
20
|
}
|
|
20
21
|
}
|
|
21
22
|
exports.ChmuWorker = ChmuWorker;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChmuWorker.js","sourceRoot":"","sources":["../../../src/integration-engine/workers/ChmuWorker.ts"],"names":[],"mappings":";;;AAAA,kCAA2C;AAC3C,sEAAoE;AACpE,2EAA6F;AAC7F,uCAAwC;
|
|
1
|
+
{"version":3,"file":"ChmuWorker.js","sourceRoot":"","sources":["../../../src/integration-engine/workers/ChmuWorker.ts"],"names":[],"mappings":";;;AAAA,kCAA2C;AAC3C,sEAAoE;AACpE,2EAA6F;AAC7F,uCAAwC;AAKxC,MAAa,UAAW,SAAQ,wBAAc;IAG1C;QACI,KAAK,EAAE,CAAC;QAHF,SAAI,GAAG,mBAAW,CAAC;QAiBtB,iBAAY,GAAG,CAAC,IAAuB,EAAQ,EAAE;YACpD,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC7C,CAAC,CAAC;QAfE,iBAAiB;QACjB,IAAI,CAAC,YAAY,CAAC,kBAAa,CAAC,OAAO,CAA2B,2CAAoB,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAClH,IAAI,CAAC,YAAY,CACb,kBAAa,CAAC,OAAO,CAAmC,2CAAoB,CAAC,gCAAgC,CAAC,CACjH,CAAC;QACF,IAAI,CAAC,YAAY,CACb,kBAAa,CAAC,OAAO,CACjB,2CAAoB,CAAC,0CAA0C,CAClE,CACJ,CAAC;IACN,CAAC;CAMJ;AAtBD,gCAsBC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export default interface IDownloadOpendataHistoricalMessage {
|
|
2
|
+
/** ISO 8601 timestamp - inclusive start of the range to download. Required; no default. */
|
|
3
|
+
from: string;
|
|
4
|
+
/** ISO 8601 timestamp - exclusive end of the range to download. Required; no default. */
|
|
5
|
+
to: string;
|
|
6
|
+
/** Station ID from chmu.stations. When set together with wsi, task runs in single-station mode. */
|
|
7
|
+
station_id?: string;
|
|
8
|
+
/** WSI identifier of the station. When set together with station_id, task runs in single-station mode. */
|
|
9
|
+
wsi?: string;
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IDownloadOpendataHistoricalMessage.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { StationsRepository } from "../../data-access/StationsRepository";
|
|
2
|
+
import { MeasurementsRepository } from "../../data-access/MeasurementsRepository";
|
|
3
|
+
import OpendataHistoricalDatasource from "../../datasources/OpendataHistoricalDatasource";
|
|
4
|
+
import { OpendataMeasurementsTransformation } from "../../transformations/OpendataMeasurementsTransformation";
|
|
5
|
+
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
6
|
+
import { AbstractTask } from "@golemio/core/dist/integration-engine";
|
|
7
|
+
import IDownloadOpendataHistoricalMessage from "../interfaces/IDownloadOpendataHistoricalMessage";
|
|
8
|
+
import { DownloadOpendataHistoricalMessageSchema } from "./DownloadOpendataHistoricalMessageSchema";
|
|
9
|
+
export default class DownloadOpendataHistoricalMeasurementsTask extends AbstractTask<IDownloadOpendataHistoricalMessage> {
|
|
10
|
+
private stationsRepository;
|
|
11
|
+
private datasource;
|
|
12
|
+
private transformation;
|
|
13
|
+
private repository;
|
|
14
|
+
private logger;
|
|
15
|
+
queueName: string;
|
|
16
|
+
protected schema: typeof DownloadOpendataHistoricalMessageSchema;
|
|
17
|
+
constructor(stationsRepository: StationsRepository, datasource: OpendataHistoricalDatasource, transformation: OpendataMeasurementsTransformation, repository: MeasurementsRepository, logger: ILogger);
|
|
18
|
+
protected execute: (data: IDownloadOpendataHistoricalMessage) => Promise<void>;
|
|
19
|
+
private generateSubtasks;
|
|
20
|
+
private processStation;
|
|
21
|
+
private generateStationMonthSubtasks;
|
|
22
|
+
private processStationMeasurements;
|
|
23
|
+
private validateRange;
|
|
24
|
+
/** Splits [from, to) into calendar-month intervals (UTC, matching DT's format); clips the first/last interval. */
|
|
25
|
+
private static generateMonthIntervals;
|
|
26
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
+
};
|
|
14
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
15
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
16
|
+
};
|
|
17
|
+
var DownloadOpendataHistoricalMeasurementsTask_1;
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
const StationsRepository_1 = require("../../data-access/StationsRepository");
|
|
20
|
+
const MeasurementsRepository_1 = require("../../data-access/MeasurementsRepository");
|
|
21
|
+
const OpendataHistoricalDatasource_1 = __importDefault(require("../../datasources/OpendataHistoricalDatasource"));
|
|
22
|
+
const ModuleContainerToken_1 = require("../../ioc/ModuleContainerToken");
|
|
23
|
+
const OpendataMeasurementsTransformation_1 = require("../../transformations/OpendataMeasurementsTransformation");
|
|
24
|
+
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
25
|
+
const QueueManager_1 = require("@golemio/core/dist/integration-engine/queueprocessors/QueueManager");
|
|
26
|
+
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
|
|
27
|
+
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
28
|
+
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
29
|
+
const DownloadOpendataHistoricalMessageSchema_1 = require("./DownloadOpendataHistoricalMessageSchema");
|
|
30
|
+
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
31
|
+
let DownloadOpendataHistoricalMeasurementsTask = DownloadOpendataHistoricalMeasurementsTask_1 = class DownloadOpendataHistoricalMeasurementsTask extends integration_engine_1.AbstractTask {
|
|
32
|
+
constructor(stationsRepository, datasource, transformation, repository, logger) {
|
|
33
|
+
super(""); // queue prefix is set by the worker
|
|
34
|
+
this.stationsRepository = stationsRepository;
|
|
35
|
+
this.datasource = datasource;
|
|
36
|
+
this.transformation = transformation;
|
|
37
|
+
this.repository = repository;
|
|
38
|
+
this.logger = logger;
|
|
39
|
+
this.queueName = "downloadOpendataHistoricalMeasurementsTask";
|
|
40
|
+
this.schema = DownloadOpendataHistoricalMessageSchema_1.DownloadOpendataHistoricalMessageSchema;
|
|
41
|
+
this.execute = async (data) => {
|
|
42
|
+
const { from, to } = this.validateRange(data);
|
|
43
|
+
try {
|
|
44
|
+
if (data.station_id && data.wsi) {
|
|
45
|
+
await this.processStation(data.station_id, data.wsi, from, to);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
await this.generateSubtasks(from, to);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
if (error instanceof golemio_errors_1.AbstractGolemioError) {
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
throw new golemio_errors_1.GeneralError(`Unable to download chmu opendata historical measurements data.`, this.constructor.name, error);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
async generateSubtasks(from, to) {
|
|
60
|
+
const stations = await this.stationsRepository.getStationsWithWsi();
|
|
61
|
+
if (stations.length === 0) {
|
|
62
|
+
this.logger.warn(`${this.constructor.name}: no stations with WSI found - skipping. Populate chmu.stations.wsi first.`);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
for (const station of stations) {
|
|
66
|
+
await this.generateStationMonthSubtasks(station.station_id, station.wsi, from, to);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async processStation(stationId, wsi, from, to) {
|
|
70
|
+
const intervals = DownloadOpendataHistoricalMeasurementsTask_1.generateMonthIntervals(from, to);
|
|
71
|
+
if (intervals.length === 1) {
|
|
72
|
+
await this.processStationMeasurements(stationId, wsi, intervals[0]);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
await this.generateStationMonthSubtasks(stationId, wsi, from, to);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async generateStationMonthSubtasks(stationId, wsi, from, to) {
|
|
79
|
+
const intervals = DownloadOpendataHistoricalMeasurementsTask_1.generateMonthIntervals(from, to);
|
|
80
|
+
for (const interval of intervals) {
|
|
81
|
+
await QueueManager_1.QueueManager.sendMessageToExchange(this.queuePrefix, this.queueName, {
|
|
82
|
+
station_id: stationId,
|
|
83
|
+
wsi,
|
|
84
|
+
from: interval.from.toISO(),
|
|
85
|
+
to: interval.to.toISO(),
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
this.logger.debug(`${this.constructor.name}: dispatched ${intervals.length} month subtasks for station ${stationId}.`);
|
|
89
|
+
}
|
|
90
|
+
async processStationMeasurements(stationId, wsi, interval) {
|
|
91
|
+
const rangeLabel = `${interval.from.toISO()} - ${interval.to.toISO()}`;
|
|
92
|
+
let stationRecords;
|
|
93
|
+
try {
|
|
94
|
+
stationRecords = await this.datasource.getData(wsi, interval.from.toJSDate());
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
throw new golemio_errors_1.GeneralError(`${this.constructor.name}: failed to fetch data for station ${stationId} (WSI: ${wsi}, range ${rangeLabel}).`);
|
|
98
|
+
}
|
|
99
|
+
if (stationRecords.length === 0) {
|
|
100
|
+
this.logger.warn(`${this.constructor.name}: no records fetched for station ${stationId} (WSI: ${wsi}), range ${rangeLabel}.`);
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
const withinRange = stationRecords.filter((r) => {
|
|
104
|
+
const dt = luxon_1.DateTime.fromISO(r.DT, { zone: "utc" });
|
|
105
|
+
return dt >= interval.from && dt < interval.to;
|
|
106
|
+
});
|
|
107
|
+
if (withinRange.length === 0) {
|
|
108
|
+
this.logger.warn(`${this.constructor.name}: all records for station ${stationId} fall outside the requested range ${rangeLabel}.`);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const records = withinRange.map((r) => ({ ...r, station_id: stationId }));
|
|
112
|
+
const transformed = await this.transformation.transform(records);
|
|
113
|
+
await this.repository.bulkUpdate(transformed);
|
|
114
|
+
this.logger.debug(`${this.constructor.name}: upserted ${transformed.length} measurements for station ${stationId}, range ${rangeLabel}.`);
|
|
115
|
+
}
|
|
116
|
+
validateRange(data) {
|
|
117
|
+
const from = luxon_1.DateTime.fromISO(data.from, { zone: "utc" });
|
|
118
|
+
const to = luxon_1.DateTime.fromISO(data.to, { zone: "utc" });
|
|
119
|
+
if (to <= from) {
|
|
120
|
+
throw new golemio_errors_1.GeneralError(`"to" (${data.to}) must be after "from" (${data.from}).`, this.constructor.name);
|
|
121
|
+
}
|
|
122
|
+
return { from, to };
|
|
123
|
+
}
|
|
124
|
+
/** Splits [from, to) into calendar-month intervals (UTC, matching DT's format); clips the first/last interval. */
|
|
125
|
+
static generateMonthIntervals(from, to) {
|
|
126
|
+
const intervals = [];
|
|
127
|
+
let cursor = from;
|
|
128
|
+
while (cursor < to) {
|
|
129
|
+
const nextMonth = cursor.startOf("month").plus({ months: 1 });
|
|
130
|
+
intervals.push({ from: cursor, to: luxon_1.DateTime.min(nextMonth, to) });
|
|
131
|
+
cursor = nextMonth;
|
|
132
|
+
}
|
|
133
|
+
return intervals;
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
DownloadOpendataHistoricalMeasurementsTask = DownloadOpendataHistoricalMeasurementsTask_1 = __decorate([
|
|
137
|
+
(0, tsyringe_1.injectable)(),
|
|
138
|
+
__param(0, (0, tsyringe_1.inject)(ModuleContainerToken_1.ModuleContainerToken.StationsRepository)),
|
|
139
|
+
__param(1, (0, tsyringe_1.inject)(ModuleContainerToken_1.ModuleContainerToken.OpendataHistoricalDatasource)),
|
|
140
|
+
__param(2, (0, tsyringe_1.inject)(ModuleContainerToken_1.ModuleContainerToken.OpendataMeasurementsTransformation)),
|
|
141
|
+
__param(3, (0, tsyringe_1.inject)(ModuleContainerToken_1.ModuleContainerToken.MeasurementsRepository)),
|
|
142
|
+
__param(4, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
|
|
143
|
+
__metadata("design:paramtypes", [StationsRepository_1.StationsRepository,
|
|
144
|
+
OpendataHistoricalDatasource_1.default,
|
|
145
|
+
OpendataMeasurementsTransformation_1.OpendataMeasurementsTransformation,
|
|
146
|
+
MeasurementsRepository_1.MeasurementsRepository, Object])
|
|
147
|
+
], DownloadOpendataHistoricalMeasurementsTask);
|
|
148
|
+
exports.default = DownloadOpendataHistoricalMeasurementsTask;
|
|
149
|
+
//# sourceMappingURL=DownloadOpendataHistoricalMeasurementsTask.js.map
|
package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DownloadOpendataHistoricalMeasurementsTask.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,6EAAwE;AACxE,qFAAgF;AAChF,kHAAwF;AACxF,yEAAoE;AACpE,iHAA4G;AAE5G,8EAAqE;AACrE,qGAAkG;AAClG,6EAA8F;AAC9F,2DAA2D;AAC3D,iEAAwE;AAExE,uGAAoG;AACpG,wEAAqE;AAQtD,IAAM,0CAA0C,kDAAhD,MAAM,0CAA2C,SAAQ,iCAAgD;IAIpH,YACqD,kBAA8C,EACpC,UAAgD,EAE3G,cAA0D,EACL,UAA0C,EACrE,MAAuB;QAEjD,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,oCAAoC;QAPU,uBAAkB,GAAlB,kBAAkB,CAAoB;QAC5B,eAAU,GAAV,UAAU,CAA8B;QAEnG,mBAAc,GAAd,cAAc,CAAoC;QACG,eAAU,GAAV,UAAU,CAAwB;QAC7D,WAAM,GAAN,MAAM,CAAS;QAT9C,cAAS,GAAW,4CAA4C,CAAC;QAC9D,WAAM,GAAG,iFAAuC,CAAC;QAajD,YAAO,GAAG,KAAK,EAAE,IAAwC,EAAiB,EAAE;YAClF,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YAE9C,IAAI,CAAC;gBACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,qCAAoB,EAAE,CAAC;oBACxC,MAAM,KAAK,CAAC;gBAChB,CAAC;gBACD,MAAM,IAAI,6BAAY,CAClB,gEAAgE,EAChE,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,KAAK,CACR,CAAC;YACN,CAAC;QACL,CAAC,CAAC;IArBF,CAAC;IAuBO,KAAK,CAAC,gBAAgB,CAAC,IAAc,EAAE,EAAY;QACvD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,EAAE,CAAC;QACpE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,4EAA4E,CACvG,CAAC;YACF,OAAO;QACX,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,GAAW,EAAE,IAAc,EAAE,EAAY;QACrF,MAAM,SAAS,GAAG,4CAA0C,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,CAAC,4BAA4B,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,4BAA4B,CAAC,SAAiB,EAAE,GAAW,EAAE,IAAc,EAAE,EAAY;QACnG,MAAM,SAAS,GAAG,4CAA0C,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAE9F,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,MAAM,2BAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE;gBACvE,UAAU,EAAE,SAAS;gBACrB,GAAG;gBACH,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE;gBAC3B,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE;aAC1B,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,gBAAgB,SAAS,CAAC,MAAM,+BAA+B,SAAS,GAAG,CAAC,CAAC;IAC3H,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,SAAiB,EAAE,GAAW,EAAE,QAAwB;QAC7F,MAAM,UAAU,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;QAEvE,IAAI,cAAc,CAAC;QACnB,IAAI,CAAC;YACD,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,6BAAY,CAClB,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,sCAAsC,SAAS,UAAU,GAAG,WAAW,UAAU,IAAI,CAChH,CAAC;QACN,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,oCAAoC,SAAS,UAAU,GAAG,YAAY,UAAU,GAAG,CAC9G,CAAC;YACF,OAAO;QACX,CAAC;QAED,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5C,MAAM,EAAE,GAAG,gBAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACnD,OAAO,EAAE,IAAI,QAAQ,CAAC,IAAI,IAAI,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,qCAAqC,UAAU,GAAG,CACnH,CAAC;YACF,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,CACb,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,WAAW,CAAC,MAAM,6BAA6B,SAAS,WAAW,UAAU,GAAG,CACzH,CAAC;IACN,CAAC;IAEO,aAAa,CAAC,IAAwC;QAC1D,MAAM,IAAI,GAAG,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,6BAAY,CAAC,SAAS,IAAI,CAAC,EAAE,2BAA2B,IAAI,CAAC,IAAI,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC5G,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACxB,CAAC;IAED,kHAAkH;IAC1G,MAAM,CAAC,sBAAsB,CAAC,IAAc,EAAE,EAAY;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,IAAI,MAAM,GAAG,IAAI,CAAC;QAClB,OAAO,MAAM,GAAG,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9D,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,gBAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAClE,MAAM,GAAG,SAAS,CAAC;QACvB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ,CAAA;AArIoB,0CAA0C;IAD9D,IAAA,qBAAU,GAAE;IAMJ,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,kBAAkB,CAAC,CAAA;IAC/C,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,4BAA4B,CAAC,CAAA;IACzD,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,kCAAkC,CAAC,CAAA;IAE/D,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,sBAAsB,CAAC,CAAA;IACnD,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;qCALoD,uCAAkB;QAChB,sCAA4B;QAEnF,uEAAkC;QACe,+CAAsB;GATlF,0CAA0C,CAqI9D;kBArIoB,0CAA0C"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import IDownloadOpendataHistoricalMessage from "../interfaces/IDownloadOpendataHistoricalMessage";
|
|
2
|
+
export declare class DownloadOpendataHistoricalMessageSchema implements IDownloadOpendataHistoricalMessage {
|
|
3
|
+
from: string;
|
|
4
|
+
to: string;
|
|
5
|
+
station_id?: string;
|
|
6
|
+
wsi?: string;
|
|
7
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.DownloadOpendataHistoricalMessageSchema = void 0;
|
|
13
|
+
const class_validator_1 = require("@golemio/core/dist/shared/class-validator");
|
|
14
|
+
const ISO8601_WITH_TIMEZONE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/;
|
|
15
|
+
class DownloadOpendataHistoricalMessageSchema {
|
|
16
|
+
}
|
|
17
|
+
exports.DownloadOpendataHistoricalMessageSchema = DownloadOpendataHistoricalMessageSchema;
|
|
18
|
+
__decorate([
|
|
19
|
+
(0, class_validator_1.IsString)(),
|
|
20
|
+
(0, class_validator_1.IsISO8601)({ strict: true }),
|
|
21
|
+
(0, class_validator_1.Matches)(ISO8601_WITH_TIMEZONE, { message: "from must be a full ISO 8601 timestamp with a timezone (e.g. Z or +01:00)" }),
|
|
22
|
+
__metadata("design:type", String)
|
|
23
|
+
], DownloadOpendataHistoricalMessageSchema.prototype, "from", void 0);
|
|
24
|
+
__decorate([
|
|
25
|
+
(0, class_validator_1.IsString)(),
|
|
26
|
+
(0, class_validator_1.IsISO8601)({ strict: true }),
|
|
27
|
+
(0, class_validator_1.Matches)(ISO8601_WITH_TIMEZONE, { message: "to must be a full ISO 8601 timestamp with a timezone (e.g. Z or +01:00)" }),
|
|
28
|
+
__metadata("design:type", String)
|
|
29
|
+
], DownloadOpendataHistoricalMessageSchema.prototype, "to", void 0);
|
|
30
|
+
__decorate([
|
|
31
|
+
(0, class_validator_1.IsOptional)(),
|
|
32
|
+
(0, class_validator_1.IsString)(),
|
|
33
|
+
__metadata("design:type", String)
|
|
34
|
+
], DownloadOpendataHistoricalMessageSchema.prototype, "station_id", void 0);
|
|
35
|
+
__decorate([
|
|
36
|
+
(0, class_validator_1.IsOptional)(),
|
|
37
|
+
(0, class_validator_1.IsString)(),
|
|
38
|
+
__metadata("design:type", String)
|
|
39
|
+
], DownloadOpendataHistoricalMessageSchema.prototype, "wsi", void 0);
|
|
40
|
+
//# sourceMappingURL=DownloadOpendataHistoricalMessageSchema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DownloadOpendataHistoricalMessageSchema.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+EAAqG;AAGrG,MAAM,qBAAqB,GAAG,kEAAkE,CAAC;AAEjG,MAAa,uCAAuC;CAkBnD;AAlBD,0FAkBC;AAdG;IAHC,IAAA,0BAAQ,GAAE;IACV,IAAA,2BAAS,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC3B,IAAA,yBAAO,EAAC,qBAAqB,EAAE,EAAE,OAAO,EAAE,2EAA2E,EAAE,CAAC;;qEAC3G;AAKd;IAHC,IAAA,0BAAQ,GAAE;IACV,IAAA,2BAAS,EAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC3B,IAAA,yBAAO,EAAC,qBAAqB,EAAE,EAAE,OAAO,EAAE,yEAAyE,EAAE,CAAC;;mEAC3G;AAIZ;IAFC,IAAA,4BAAU,GAAE;IACZ,IAAA,0BAAQ,GAAE;;2EACS;AAIpB;IAFC,IAAA,4BAAU,GAAE;IACZ,IAAA,0BAAQ,GAAE;;oEACE"}
|
|
@@ -18,5 +18,5 @@ export default class DownloadOpendataMeasurementsTask extends AbstractTask<IDown
|
|
|
18
18
|
protected execute: (data: IDownloadOpendataMessage) => Promise<void>;
|
|
19
19
|
private generateSubtasks;
|
|
20
20
|
private processStationMeasurements;
|
|
21
|
-
private
|
|
21
|
+
private validateDate;
|
|
22
22
|
}
|
|
@@ -38,7 +38,7 @@ let DownloadOpendataMeasurementsTask = class DownloadOpendataMeasurementsTask ex
|
|
|
38
38
|
this.queueName = "downloadOpendataMeasurementsTask";
|
|
39
39
|
this.schema = DownloadOpendataMessageSchema_1.DownloadOpendataMessageSchema;
|
|
40
40
|
this.execute = async (data) => {
|
|
41
|
-
const { date } = this.
|
|
41
|
+
const { date } = this.validateDate(data);
|
|
42
42
|
try {
|
|
43
43
|
if (data.wsi && data.station_id) {
|
|
44
44
|
await this.processStationMeasurements(data.station_id, data.wsi, new Date(date));
|
|
@@ -51,7 +51,7 @@ let DownloadOpendataMeasurementsTask = class DownloadOpendataMeasurementsTask ex
|
|
|
51
51
|
if (error instanceof golemio_errors_1.AbstractGolemioError) {
|
|
52
52
|
throw error;
|
|
53
53
|
}
|
|
54
|
-
|
|
54
|
+
throw new golemio_errors_1.GeneralError(`Unable to download chmu opendata measurements data.`, this.constructor.name, error);
|
|
55
55
|
}
|
|
56
56
|
};
|
|
57
57
|
}
|
|
@@ -87,13 +87,13 @@ let DownloadOpendataMeasurementsTask = class DownloadOpendataMeasurementsTask ex
|
|
|
87
87
|
await this.repository.bulkUpdate(transformed);
|
|
88
88
|
this.logger.debug(`${this.constructor.name}: upserted ${transformed.length} measurements for station ${stationId}.`);
|
|
89
89
|
}
|
|
90
|
-
|
|
90
|
+
validateDate(data) {
|
|
91
91
|
const dt = data.date ? luxon_1.DateTime.fromISO(data.date) : luxon_1.DateTime.now();
|
|
92
92
|
const ageInDays = luxon_1.DateTime.now().startOf("day").diff(dt.startOf("day"), "days").days;
|
|
93
93
|
if (ageInDays > 2) {
|
|
94
|
-
throw new
|
|
94
|
+
throw new golemio_errors_1.GeneralError(`Date "${data.date}" is outside the 3-day retention window`, this.constructor.name);
|
|
95
95
|
}
|
|
96
|
-
return {
|
|
96
|
+
return { date: dt.toISODate() };
|
|
97
97
|
}
|
|
98
98
|
};
|
|
99
99
|
DownloadOpendataMeasurementsTask = __decorate([
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DownloadOpendataMeasurementsTask.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,6EAAwE;AACxE,qFAAgF;AAChF,8FAAoE;AACpE,yEAAoE;AACpE,iHAA4G;AAE5G,8EAAqE;AACrE,qGAAkG;AAClG,6EAA8F;AAC9F,2DAA2D;AAC3D,iEAAwE;AAExE,mFAAgF;AAChF,wEAAqE;AAGtD,IAAM,gCAAgC,GAAtC,MAAM,gCAAiC,SAAQ,iCAAsC;IAIhG,YACqD,kBAA8C,EAC9C,UAAsC,EAEvF,cAA0D,EACL,UAA0C,EACrE,MAAuB;QAEjD,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,oCAAoC;QAPU,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAoB;QAE/E,mBAAc,GAAd,cAAc,CAAoC;QACG,eAAU,GAAV,UAAU,CAAwB;QAC7D,WAAM,GAAN,MAAM,CAAS;QAT9C,cAAS,GAAW,kCAAkC,CAAC;QACpD,WAAM,GAAG,6DAA6B,CAAC;QAavC,YAAO,GAAG,KAAK,EAAE,IAA8B,EAAiB,EAAE;YACxE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"DownloadOpendataMeasurementsTask.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,6EAAwE;AACxE,qFAAgF;AAChF,8FAAoE;AACpE,yEAAoE;AACpE,iHAA4G;AAE5G,8EAAqE;AACrE,qGAAkG;AAClG,6EAA8F;AAC9F,2DAA2D;AAC3D,iEAAwE;AAExE,mFAAgF;AAChF,wEAAqE;AAGtD,IAAM,gCAAgC,GAAtC,MAAM,gCAAiC,SAAQ,iCAAsC;IAIhG,YACqD,kBAA8C,EAC9C,UAAsC,EAEvF,cAA0D,EACL,UAA0C,EACrE,MAAuB;QAEjD,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,oCAAoC;QAPU,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAoB;QAE/E,mBAAc,GAAd,cAAc,CAAoC;QACG,eAAU,GAAV,UAAU,CAAwB;QAC7D,WAAM,GAAN,MAAM,CAAS;QAT9C,cAAS,GAAW,kCAAkC,CAAC;QACpD,WAAM,GAAG,6DAA6B,CAAC;QAavC,YAAO,GAAG,KAAK,EAAE,IAA8B,EAAiB,EAAE;YACxE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAEzC,IAAI,CAAC;gBACD,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC9B,MAAM,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBACrF,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBACtC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,qCAAoB,EAAE,CAAC;oBACxC,MAAM,KAAK,CAAC;gBAChB,CAAC;gBACD,MAAM,IAAI,6BAAY,CAAC,qDAAqD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAChH,CAAC;QACL,CAAC,CAAC;IAjBF,CAAC;IAmBO,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACvC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,EAAE,CAAC;QACpE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CACZ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,4EAA4E,CACvG,CAAC;YACF,OAAO;QACX,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,2BAAY,CAAC,qBAAqB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE;gBACvE,IAAI;gBACJ,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,GAAG,EAAE,OAAO,CAAC,GAAG;aACnB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,gBAAgB,QAAQ,CAAC,MAAM,sBAAsB,IAAI,GAAG,CAAC,CAAC;IAC5G,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,SAAiB,EAAE,GAAW,EAAE,IAAU;QAC/E,IAAI,cAAc,CAAC;QACnB,IAAI,CAAC;YACD,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,6BAAY,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,sCAAsC,SAAS,UAAU,GAAG,IAAI,CAAC,CAAC;QACrH,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,oCAAoC,SAAS,UAAU,GAAG,IAAI,CAAC,CAAC;YACzG,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;QAC7E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,WAAW,CAAC,MAAM,6BAA6B,SAAS,GAAG,CAAC,CAAC;IACzH,CAAC;IAEO,YAAY,CAAC,IAA8B;QAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAQ,CAAC,GAAG,EAAE,CAAC;QAEpE,MAAM,SAAS,GAAG,gBAAQ,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;QACrF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,6BAAY,CAAC,SAAS,IAAI,CAAC,IAAI,yCAAyC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/G,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,SAAS,EAAG,EAAE,CAAC;IACrC,CAAC;CACJ,CAAA;AAhFoB,gCAAgC;IADpD,IAAA,qBAAU,GAAE;IAMJ,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,kBAAkB,CAAC,CAAA;IAC/C,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,kBAAkB,CAAC,CAAA;IAC/C,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,kCAAkC,CAAC,CAAA;IAE/D,WAAA,IAAA,iBAAM,EAAC,2CAAoB,CAAC,sBAAsB,CAAC,CAAA;IACnD,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,MAAM,CAAC,CAAA;qCALoD,uCAAkB;QAC1B,4BAAkB;QAE/D,uEAAkC;QACe,+CAAsB;GATlF,gCAAgC,CAgFpD;kBAhFoB,gCAAgC"}
|
package/docs/asyncapi.yaml
CHANGED
|
@@ -17,7 +17,7 @@ channels:
|
|
|
17
17
|
$ref: "#/components/messages/chmu.DownloadMeasurementsMessage"
|
|
18
18
|
chmu.downloadOpendataMeasurementsTask:
|
|
19
19
|
address: dataplatform.chmu.downloadOpendataMeasurementsTask
|
|
20
|
-
description: Downloads CHMU 10-minute
|
|
20
|
+
description: Downloads CHMU 10-minute opendata for all stations with a WSI identifier.
|
|
21
21
|
bindings:
|
|
22
22
|
amqp:
|
|
23
23
|
is: queue
|
|
@@ -26,6 +26,17 @@ channels:
|
|
|
26
26
|
messages:
|
|
27
27
|
chmu.DownloadOpendataMessage:
|
|
28
28
|
$ref: "#/components/messages/chmu.DownloadOpendataMessage"
|
|
29
|
+
chmu.downloadOpendataHistoricalMeasurementsTask:
|
|
30
|
+
address: dataplatform.chmu.downloadOpendataHistoricalMeasurementsTask
|
|
31
|
+
description: Downloads historical 10-minute opendata for an arbitrary date range.
|
|
32
|
+
bindings:
|
|
33
|
+
amqp:
|
|
34
|
+
is: queue
|
|
35
|
+
queue:
|
|
36
|
+
durable: true
|
|
37
|
+
messages:
|
|
38
|
+
chmu.DownloadOpendataHistoricalMessage:
|
|
39
|
+
$ref: "#/components/messages/chmu.DownloadOpendataHistoricalMessage"
|
|
29
40
|
operations:
|
|
30
41
|
chmu.downloadMeasurementsTask:
|
|
31
42
|
action: "send"
|
|
@@ -35,6 +46,10 @@ operations:
|
|
|
35
46
|
action: "send"
|
|
36
47
|
channel:
|
|
37
48
|
$ref: "#/channels/chmu.downloadOpendataMeasurementsTask"
|
|
49
|
+
chmu.downloadOpendataHistoricalMeasurementsTask:
|
|
50
|
+
action: "send"
|
|
51
|
+
channel:
|
|
52
|
+
$ref: "#/channels/chmu.downloadOpendataHistoricalMeasurementsTask"
|
|
38
53
|
components:
|
|
39
54
|
messages:
|
|
40
55
|
chmu.DownloadMeasurementsMessage:
|
|
@@ -65,3 +80,31 @@ components:
|
|
|
65
80
|
nullable: true
|
|
66
81
|
description: ČHMÚ WSI station identifier.
|
|
67
82
|
additionalProperties: false
|
|
83
|
+
chmu.DownloadOpendataHistoricalMessage:
|
|
84
|
+
payload:
|
|
85
|
+
type: object
|
|
86
|
+
properties:
|
|
87
|
+
"from":
|
|
88
|
+
type: string
|
|
89
|
+
format: date-time
|
|
90
|
+
description: >
|
|
91
|
+
ISO 8601 timestamp with an explicit timezone (Z or ±HH:MM) - inclusive start of the range
|
|
92
|
+
to download. Required; no default.
|
|
93
|
+
"to":
|
|
94
|
+
type: string
|
|
95
|
+
format: date-time
|
|
96
|
+
description: >
|
|
97
|
+
ISO 8601 timestamp with an explicit timezone (Z or ±HH:MM) - exclusive end of the range
|
|
98
|
+
to download. Required; no default.
|
|
99
|
+
"station_id":
|
|
100
|
+
type: string
|
|
101
|
+
nullable: true
|
|
102
|
+
description: Station identifier. Together with wsi, runs in single-station mode.
|
|
103
|
+
"wsi":
|
|
104
|
+
type: string
|
|
105
|
+
nullable: true
|
|
106
|
+
description: ČHMÚ WSI station identifier. Together with station_id, runs in single-station mode.
|
|
107
|
+
required:
|
|
108
|
+
- from
|
|
109
|
+
- to
|
|
110
|
+
additionalProperties: false
|
|
@@ -45,6 +45,19 @@ Data z obou zdrojů se ukládají do stejné tabulky `chmu.measurements` pomocí
|
|
|
45
45
|
- poznámka
|
|
46
46
|
- Zpracovány jsou pouze stanice s vyplněným sloupcem `wsi` v `chmu.stations`; ostatní jsou přeskočeny.
|
|
47
47
|
|
|
48
|
+
#### *OpendataHistoricalDatasource* (jednorázový download historických dat)
|
|
49
|
+
|
|
50
|
+
- zdroj dat
|
|
51
|
+
- ČHMÚ opendata - historický 10min JSON archiv
|
|
52
|
+
- URL: `https://opendata.chmi.cz/meteorology/climate/historical/data/10min/{rok}/10m-{WSI}-{YYYYMM}.json`
|
|
53
|
+
- jeden soubor = jedna stanice × jeden měsíc, všechny veličiny dané stanice (stejně jako živý soubor)
|
|
54
|
+
- formát dat
|
|
55
|
+
- stejný jako *OpendataDatasource* výše
|
|
56
|
+
- frekvence stahování
|
|
57
|
+
- jednorázový batch (ne cron)
|
|
58
|
+
- název RabbitMQ fronty
|
|
59
|
+
- `chmu.downloadOpendataHistoricalMeasurementsTask`
|
|
60
|
+
|
|
48
61
|
## Zpracování dat / transformace
|
|
49
62
|
|
|
50
63
|
### *ChmuWorker*
|
|
@@ -101,6 +114,26 @@ Data z obou zdrojů se ukládají do stejné tabulky `chmu.measurements` pomocí
|
|
|
101
114
|
- data modely
|
|
102
115
|
- [MeasurementModel](https://gitlab.com/operator-ict/golemio/code/modules/chmu/-/blob/development/src/schema-definitions/definitions/models/MeasurementModel.ts)
|
|
103
116
|
|
|
117
|
+
#### *DownloadOpendataHistoricalMeasurementsTask*
|
|
118
|
+
|
|
119
|
+
- vstupní RabbitMQ fronta
|
|
120
|
+
- `chmu.downloadOpendataHistoricalMeasurementsTask`
|
|
121
|
+
- parametry `from` / `to` - povinné ISO 8601 timestampy s časovou zónou (`Z` nebo `±HH:MM`); volitelně `station_id` + `wsi` pro omezení na jednu stanici
|
|
122
|
+
- bez `station_id`/`wsi` rozešle subtasky pro všechny stanice; se `station_id`+`wsi` a rozsahem přes více měsíců rozešle subtasky po měsících; rozsah v rámci jednoho měsíce se zpracuje rovnou (leaf mode)
|
|
123
|
+
- tok zpracování
|
|
124
|
+
1. `StationsRepository` - stejné jako u živého pipeline (stanice s vyplněným WSI)
|
|
125
|
+
2. `OpendataHistoricalDatasource` - jeden fetch na stanici × měsíc (měsíc určen z `from`)
|
|
126
|
+
3. Přiřazení `station_id` dle WSI -> `station_id` mappingu
|
|
127
|
+
4. Filtr `from <= DT < to` dle rozsahu dílčí zprávy
|
|
128
|
+
5. `OpendataMeasurementsTransformation` - stejná jako u živého pipeline (pivot long->wide, `data_source = 'opendata'`, žádná hodinová agregace - ukládá se ve stejné 10minutové granularitě jako živá data)
|
|
129
|
+
6. `MeasurementsRepository.bulkUpdate` - stejné UPSERT jako u živého pipeline; `data_source` je součástí primárního klíče, takže `opendata` řádky nikdy nekolidují s `ftp` řádky
|
|
130
|
+
- mapování ELEMENT -> DB sloupec
|
|
131
|
+
- stejné jako u *DownloadOpendataMeasurementsTask* výše
|
|
132
|
+
- data modely
|
|
133
|
+
- [MeasurementModel](https://gitlab.com/operator-ict/golemio/code/modules/chmu/-/blob/development/src/schema-definitions/definitions/models/MeasurementModel.ts)
|
|
134
|
+
- poznámka
|
|
135
|
+
- Rozsah (`from`/`to`) určuje výhradně volající zpráva.
|
|
136
|
+
|
|
104
137
|
## Uložení dat
|
|
105
138
|
|
|
106
139
|
### Obecné
|