@golemio/chmu 1.1.1-dev.2674913750 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/integration-engine/data-access/MeasurementsRepository.js +1 -1
- package/dist/integration-engine/data-access/MeasurementsRepository.js.map +1 -1
- package/dist/integration-engine/ioc/Di.js +0 -12
- package/dist/integration-engine/ioc/Di.js.map +1 -1
- package/dist/integration-engine/ioc/ModuleContainerToken.d.ts +0 -6
- package/dist/integration-engine/ioc/ModuleContainerToken.js +0 -8
- package/dist/integration-engine/ioc/ModuleContainerToken.js.map +1 -1
- package/dist/integration-engine/transformations/MeasurementsTransformation.js +0 -18
- package/dist/integration-engine/transformations/MeasurementsTransformation.js.map +1 -1
- package/dist/integration-engine/workers/ChmuWorker.js +0 -2
- package/dist/integration-engine/workers/ChmuWorker.js.map +1 -1
- package/dist/integration-engine/workers/tasks/DownloadMeasurementsTask.js +2 -2
- package/dist/integration-engine/workers/tasks/DownloadMeasurementsTask.js.map +1 -1
- package/dist/schema-definitions/definitions/interfaces/IMeasurementDto.d.ts +0 -18
- package/dist/schema-definitions/definitions/models/MeasurementModel.d.ts +0 -18
- package/dist/schema-definitions/definitions/models/MeasurementModel.js +1 -37
- package/dist/schema-definitions/definitions/models/MeasurementModel.js.map +1 -1
- package/docs/asyncapi.yaml +3 -78
- package/docs/implementation_documentation.md +23 -114
- package/package.json +4 -4
- package/db/migrations/postgresql/20260619120000-opendata-columns.js +0 -49
- package/db/migrations/postgresql/sqls/20260619120000-opendata-columns-down.sql +0 -36
- package/db/migrations/postgresql/sqls/20260619120000-opendata-columns-up.sql +0 -40
- package/dist/integration-engine/data-access/StationsRepository.d.ts +0 -8
- package/dist/integration-engine/data-access/StationsRepository.js +0 -57
- package/dist/integration-engine/data-access/StationsRepository.js.map +0 -1
- package/dist/integration-engine/datasources/AbstractOpendataDatasource.d.ts +0 -19
- package/dist/integration-engine/datasources/AbstractOpendataDatasource.js +0 -47
- package/dist/integration-engine/datasources/AbstractOpendataDatasource.js.map +0 -1
- package/dist/integration-engine/datasources/OpendataDatasource.d.ts +0 -8
- package/dist/integration-engine/datasources/OpendataDatasource.js +0 -37
- package/dist/integration-engine/datasources/OpendataDatasource.js.map +0 -1
- package/dist/integration-engine/datasources/OpendataHistoricalDatasource.d.ts +0 -8
- package/dist/integration-engine/datasources/OpendataHistoricalDatasource.js +0 -38
- package/dist/integration-engine/datasources/OpendataHistoricalDatasource.js.map +0 -1
- package/dist/integration-engine/transformations/OpendataMeasurementsTransformation.d.ts +0 -15
- package/dist/integration-engine/transformations/OpendataMeasurementsTransformation.js +0 -128
- package/dist/integration-engine/transformations/OpendataMeasurementsTransformation.js.map +0 -1
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.d.ts +0 -10
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.js +0 -3
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.js.map +0 -1
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataMessage.d.ts +0 -8
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataMessage.js +0 -3
- package/dist/integration-engine/workers/interfaces/IDownloadOpendataMessage.js.map +0 -1
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.d.ts +0 -26
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.js +0 -149
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.js.map +0 -1
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.d.ts +0 -7
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.js +0 -40
- package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMessageSchema.js.map +0 -1
- package/dist/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.d.ts +0 -22
- package/dist/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.js +0 -112
- package/dist/integration-engine/workers/tasks/DownloadOpendataMeasurementsTask.js.map +0 -1
- package/dist/integration-engine/workers/tasks/DownloadOpendataMessageSchema.d.ts +0 -6
- package/dist/integration-engine/workers/tasks/DownloadOpendataMessageSchema.js +0 -33
- package/dist/integration-engine/workers/tasks/DownloadOpendataMessageSchema.js.map +0 -1
- package/dist/schema-definitions/datasources/OpendataSchemaProvider.d.ts +0 -55
- package/dist/schema-definitions/datasources/OpendataSchemaProvider.js +0 -47
- package/dist/schema-definitions/datasources/OpendataSchemaProvider.js.map +0 -1
- package/dist/schema-definitions/datasources/interfaces/IInputOpendataRecord.d.ts +0 -15
- package/dist/schema-definitions/datasources/interfaces/IInputOpendataRecord.js +0 -3
- package/dist/schema-definitions/datasources/interfaces/IInputOpendataRecord.js.map +0 -1
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var dbm;
|
|
4
|
-
var type;
|
|
5
|
-
var seed;
|
|
6
|
-
var fs = require('fs');
|
|
7
|
-
var path = require('path');
|
|
8
|
-
var Promise;
|
|
9
|
-
|
|
10
|
-
exports.setup = function(options, seedLink) {
|
|
11
|
-
dbm = options.dbmigrate;
|
|
12
|
-
type = dbm.dataType;
|
|
13
|
-
seed = seedLink;
|
|
14
|
-
Promise = options.Promise;
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
exports.up = function(db) {
|
|
18
|
-
var filePath = path.join(__dirname, 'sqls', '20260619120000-opendata-columns-up.sql');
|
|
19
|
-
return new Promise( function( resolve, reject ) {
|
|
20
|
-
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
21
|
-
if (err) return reject(err);
|
|
22
|
-
console.log('received data: ' + data);
|
|
23
|
-
|
|
24
|
-
resolve(data);
|
|
25
|
-
});
|
|
26
|
-
})
|
|
27
|
-
.then(function(data) {
|
|
28
|
-
return db.runSql(data);
|
|
29
|
-
});
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
exports.down = function(db) {
|
|
33
|
-
var filePath = path.join(__dirname, 'sqls', '20260619120000-opendata-columns-down.sql');
|
|
34
|
-
return new Promise( function( resolve, reject ) {
|
|
35
|
-
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
36
|
-
if (err) return reject(err);
|
|
37
|
-
console.log('received data: ' + data);
|
|
38
|
-
|
|
39
|
-
resolve(data);
|
|
40
|
-
});
|
|
41
|
-
})
|
|
42
|
-
.then(function(data) {
|
|
43
|
-
return db.runSql(data);
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
exports._meta = {
|
|
48
|
-
"version": 1
|
|
49
|
-
};
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
ALTER TABLE measurements
|
|
2
|
-
DROP CONSTRAINT measurements_pk,
|
|
3
|
-
ADD CONSTRAINT measurements_pk PRIMARY KEY (measured_at, station_id);
|
|
4
|
-
|
|
5
|
-
ALTER TABLE measurements
|
|
6
|
-
DROP COLUMN IF EXISTS air_temp_max,
|
|
7
|
-
DROP COLUMN IF EXISTS air_temp_min,
|
|
8
|
-
DROP COLUMN IF EXISTS air_temp_ground,
|
|
9
|
-
DROP COLUMN IF EXISTS wind_speed_avg,
|
|
10
|
-
DROP COLUMN IF EXISTS wind_dir_avg,
|
|
11
|
-
DROP COLUMN IF EXISTS wind_gust,
|
|
12
|
-
DROP COLUMN IF EXISTS wind_gust_dir,
|
|
13
|
-
DROP COLUMN IF EXISTS wind_gust_time,
|
|
14
|
-
DROP COLUMN IF EXISTS snow_depth,
|
|
15
|
-
DROP COLUMN IF EXISTS snow_water,
|
|
16
|
-
DROP COLUMN IF EXISTS radiation_glob,
|
|
17
|
-
DROP COLUMN IF EXISTS radiation_uv,
|
|
18
|
-
DROP COLUMN IF EXISTS soil_temp_5cm,
|
|
19
|
-
DROP COLUMN IF EXISTS soil_temp_10cm,
|
|
20
|
-
DROP COLUMN IF EXISTS soil_temp_20cm,
|
|
21
|
-
DROP COLUMN IF EXISTS soil_temp_50cm,
|
|
22
|
-
DROP COLUMN IF EXISTS soil_temp_100cm,
|
|
23
|
-
DROP COLUMN IF EXISTS data_source;
|
|
24
|
-
|
|
25
|
-
ALTER TABLE stations
|
|
26
|
-
DROP COLUMN IF EXISTS wsi,
|
|
27
|
-
DROP COLUMN IF EXISTS air_temp_max,
|
|
28
|
-
DROP COLUMN IF EXISTS air_temp_min,
|
|
29
|
-
DROP COLUMN IF EXISTS air_temp_ground,
|
|
30
|
-
DROP COLUMN IF EXISTS wind_speed_avg,
|
|
31
|
-
DROP COLUMN IF EXISTS wind_gust,
|
|
32
|
-
DROP COLUMN IF EXISTS snow_depth,
|
|
33
|
-
DROP COLUMN IF EXISTS snow_water,
|
|
34
|
-
DROP COLUMN IF EXISTS radiation_glob,
|
|
35
|
-
DROP COLUMN IF EXISTS radiation_uv,
|
|
36
|
-
DROP COLUMN IF EXISTS soil_temp;
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
-- Add WSI identifier and boolean flags for new measurement elements to stations
|
|
2
|
-
-- NOTE: Populate chmu.stations.wsi and the boolean flag columns (manually) before running the opendata pipeline.
|
|
3
|
-
ALTER TABLE stations
|
|
4
|
-
ADD COLUMN wsi varchar NULL,
|
|
5
|
-
ADD COLUMN air_temp_max boolean NULL,
|
|
6
|
-
ADD COLUMN air_temp_min boolean NULL,
|
|
7
|
-
ADD COLUMN air_temp_ground boolean NULL,
|
|
8
|
-
ADD COLUMN wind_speed_avg boolean NULL,
|
|
9
|
-
ADD COLUMN wind_gust boolean NULL,
|
|
10
|
-
ADD COLUMN snow_depth boolean NULL,
|
|
11
|
-
ADD COLUMN snow_water boolean NULL,
|
|
12
|
-
ADD COLUMN radiation_glob boolean NULL,
|
|
13
|
-
ADD COLUMN radiation_uv boolean NULL,
|
|
14
|
-
ADD COLUMN soil_temp boolean NULL;
|
|
15
|
-
|
|
16
|
-
-- Add new measurement columns and data_source to measurements
|
|
17
|
-
ALTER TABLE measurements
|
|
18
|
-
ADD COLUMN air_temp_max double precision NULL,
|
|
19
|
-
ADD COLUMN air_temp_min double precision NULL,
|
|
20
|
-
ADD COLUMN air_temp_ground double precision NULL,
|
|
21
|
-
ADD COLUMN wind_speed_avg double precision NULL,
|
|
22
|
-
ADD COLUMN wind_dir_avg double precision NULL,
|
|
23
|
-
ADD COLUMN wind_gust double precision NULL,
|
|
24
|
-
ADD COLUMN wind_gust_dir double precision NULL,
|
|
25
|
-
ADD COLUMN wind_gust_time integer NULL,
|
|
26
|
-
ADD COLUMN snow_depth double precision NULL,
|
|
27
|
-
ADD COLUMN snow_water double precision NULL,
|
|
28
|
-
ADD COLUMN radiation_glob double precision NULL,
|
|
29
|
-
ADD COLUMN radiation_uv double precision NULL,
|
|
30
|
-
ADD COLUMN soil_temp_5cm double precision NULL,
|
|
31
|
-
ADD COLUMN soil_temp_10cm double precision NULL,
|
|
32
|
-
ADD COLUMN soil_temp_20cm double precision NULL,
|
|
33
|
-
ADD COLUMN soil_temp_50cm double precision NULL,
|
|
34
|
-
ADD COLUMN soil_temp_100cm double precision NULL,
|
|
35
|
-
ADD COLUMN data_source varchar(10) NOT NULL DEFAULT 'ftp';
|
|
36
|
-
|
|
37
|
-
-- Add data_source to PK to prevent FTP/opendata record collision at shared timestamps
|
|
38
|
-
ALTER TABLE measurements
|
|
39
|
-
DROP CONSTRAINT measurements_pk,
|
|
40
|
-
ADD CONSTRAINT measurements_pk PRIMARY KEY (measured_at, station_id, data_source);
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine";
|
|
2
|
-
export declare class StationsRepository extends PostgresModel implements IModel {
|
|
3
|
-
constructor();
|
|
4
|
-
getStationsWithWsi: () => Promise<Array<{
|
|
5
|
-
station_id: string;
|
|
6
|
-
wsi: string;
|
|
7
|
-
}>>;
|
|
8
|
-
}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.StationsRepository = void 0;
|
|
13
|
-
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
|
|
14
|
-
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
15
|
-
const sequelize_1 = require("@golemio/core/dist/shared/sequelize");
|
|
16
|
-
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
17
|
-
const const_1 = require("../../const");
|
|
18
|
-
class StationWsiModel extends sequelize_1.Model {
|
|
19
|
-
}
|
|
20
|
-
StationWsiModel.attributeModel = {
|
|
21
|
-
station_id: { type: sequelize_1.DataTypes.STRING, primaryKey: true },
|
|
22
|
-
wsi: { type: sequelize_1.DataTypes.STRING, allowNull: true },
|
|
23
|
-
};
|
|
24
|
-
let StationsRepository = class StationsRepository extends integration_engine_1.PostgresModel {
|
|
25
|
-
constructor() {
|
|
26
|
-
super("ChmuStationsRepository", {
|
|
27
|
-
outputSequelizeAttributes: StationWsiModel.attributeModel,
|
|
28
|
-
pgTableName: "stations",
|
|
29
|
-
pgSchema: const_1.SCHEMA_NAME,
|
|
30
|
-
savingType: "insertOrUpdate",
|
|
31
|
-
}, new golemio_validator_1.JSONSchemaValidator("ChmuStationsValidator", {
|
|
32
|
-
type: "array",
|
|
33
|
-
items: {
|
|
34
|
-
type: "object",
|
|
35
|
-
properties: {
|
|
36
|
-
station_id: { type: "string" },
|
|
37
|
-
wsi: { oneOf: [{ type: "string" }, { type: "null", nullable: true }] },
|
|
38
|
-
},
|
|
39
|
-
required: ["station_id"],
|
|
40
|
-
},
|
|
41
|
-
}));
|
|
42
|
-
this.getStationsWithWsi = async () => {
|
|
43
|
-
const rows = await this.sequelizeModel.findAll({
|
|
44
|
-
where: { wsi: { [sequelize_1.Op.not]: null } },
|
|
45
|
-
attributes: ["station_id", "wsi"],
|
|
46
|
-
raw: true,
|
|
47
|
-
});
|
|
48
|
-
return rows;
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
};
|
|
52
|
-
exports.StationsRepository = StationsRepository;
|
|
53
|
-
exports.StationsRepository = StationsRepository = __decorate([
|
|
54
|
-
(0, tsyringe_1.injectable)(),
|
|
55
|
-
__metadata("design:paramtypes", [])
|
|
56
|
-
], StationsRepository);
|
|
57
|
-
//# sourceMappingURL=StationsRepository.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"StationsRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/StationsRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8EAA8E;AAC9E,mFAAkF;AAClF,mEAA4F;AAC5F,iEAAgE;AAChE,uCAAwC;AAOxC,MAAM,eAAgB,SAAQ,iBAAkB;;AAI9B,8BAAc,GAAkD;IAC1E,UAAU,EAAE,EAAE,IAAI,EAAE,qBAAS,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE;IACxD,GAAG,EAAE,EAAE,IAAI,EAAE,qBAAS,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;CACnD,CAAC;AAIC,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,kCAAa;IACjD;QACI,KAAK,CACD,wBAAwB,EACxB;YACI,yBAAyB,EAAE,eAAe,CAAC,cAAc;YACzD,WAAW,EAAE,UAAU;YACvB,QAAQ,EAAE,mBAAW;YACrB,UAAU,EAAE,gBAAgB;SAC/B,EACD,IAAI,uCAAmB,CAAC,uBAAuB,EAAE;YAC7C,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACH,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACR,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC9B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;iBACzE;gBACD,QAAQ,EAAE,CAAC,YAAY,CAAC;aAC3B;SACJ,CAAC,CACL,CAAC;QAGC,uBAAkB,GAAG,KAAK,IAAyD,EAAE;YACxF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAkB;gBAC5D,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,cAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE;gBAClC,UAAU,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC;gBACjC,GAAG,EAAE,IAAI;aACZ,CAAC,CAAC;YACH,OAAO,IAA6D,CAAC;QACzE,CAAC,CAAC;IATF,CAAC;CAUJ,CAAA;AAhCY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,qBAAU,GAAE;;GACA,kBAAkB,CAgC9B"}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
/** Raw record shape shared by both the live (daily) and historical (monthly) ČHMÚ opendata sources. */
|
|
2
|
-
export interface IRawOpendataRecord {
|
|
3
|
-
STATION: string;
|
|
4
|
-
ELEMENT: string;
|
|
5
|
-
DT: string;
|
|
6
|
-
VAL: number | "";
|
|
7
|
-
FLAG: string;
|
|
8
|
-
QUALITY: number;
|
|
9
|
-
}
|
|
10
|
-
export declare abstract class AbstractOpendataDatasource {
|
|
11
|
-
abstract name: string;
|
|
12
|
-
private readonly responseValidator;
|
|
13
|
-
private readonly recordValidator;
|
|
14
|
-
protected abstract buildUrl(wsi: string, date: Date): string;
|
|
15
|
-
getData(wsi: string, date?: Date): Promise<IRawOpendataRecord[]>;
|
|
16
|
-
private getDatasource;
|
|
17
|
-
private parseHeader;
|
|
18
|
-
private mapToRecords;
|
|
19
|
-
}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.AbstractOpendataDatasource = void 0;
|
|
7
|
-
const datasources_1 = require("@golemio/core/dist/integration-engine/datasources");
|
|
8
|
-
const HTTPFetchProtocolStrategy_1 = require("@golemio/core/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy");
|
|
9
|
-
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
10
|
-
const OpendataSchemaProvider_1 = __importDefault(require("../../schema-definitions/datasources/OpendataSchemaProvider"));
|
|
11
|
-
const REQUIRED_COLUMNS = ["STATION", "ELEMENT", "DT", "VAL", "FLAG", "QUALITY"];
|
|
12
|
-
class AbstractOpendataDatasource {
|
|
13
|
-
constructor() {
|
|
14
|
-
this.responseValidator = new golemio_validator_1.JSONSchemaValidator("OpendataResponseValidator", OpendataSchemaProvider_1.default.getResponseSchema());
|
|
15
|
-
this.recordValidator = new golemio_validator_1.JSONSchemaValidator("OpendataRecordValidator", OpendataSchemaProvider_1.default.get());
|
|
16
|
-
}
|
|
17
|
-
async getData(wsi, date = new Date()) {
|
|
18
|
-
const { header, values } = await this.getDatasource(this.buildUrl(wsi, date)).getAll();
|
|
19
|
-
const colIndex = this.parseHeader(header);
|
|
20
|
-
const records = this.mapToRecords(values, colIndex);
|
|
21
|
-
await this.recordValidator.Validate(records);
|
|
22
|
-
return records;
|
|
23
|
-
}
|
|
24
|
-
getDatasource(url) {
|
|
25
|
-
return new datasources_1.DataSource("OpendataHttpDataSource", new HTTPFetchProtocolStrategy_1.HTTPFetchProtocolStrategy({ url, method: "GET", responseType: "json" }), new datasources_1.JSONDataTypeStrategy({ resultsPath: "data.data" }), this.responseValidator);
|
|
26
|
-
}
|
|
27
|
-
parseHeader(header) {
|
|
28
|
-
const colIndex = Object.fromEntries(header.split(",").map((c, i) => [c, i]));
|
|
29
|
-
const missing = REQUIRED_COLUMNS.filter((c) => colIndex[c] === undefined);
|
|
30
|
-
if (missing.length > 0) {
|
|
31
|
-
throw new Error(`${this.constructor.name}: missing columns in header: ${missing.join(", ")}`);
|
|
32
|
-
}
|
|
33
|
-
return colIndex;
|
|
34
|
-
}
|
|
35
|
-
mapToRecords(values, colIndex) {
|
|
36
|
-
return values.map((row) => ({
|
|
37
|
-
STATION: row[colIndex["STATION"]],
|
|
38
|
-
ELEMENT: row[colIndex["ELEMENT"]],
|
|
39
|
-
DT: row[colIndex["DT"]],
|
|
40
|
-
VAL: row[colIndex["VAL"]],
|
|
41
|
-
FLAG: row[colIndex["FLAG"]],
|
|
42
|
-
QUALITY: row[colIndex["QUALITY"]],
|
|
43
|
-
}));
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
exports.AbstractOpendataDatasource = AbstractOpendataDatasource;
|
|
47
|
-
//# sourceMappingURL=AbstractOpendataDatasource.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractOpendataDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/AbstractOpendataDatasource.ts"],"names":[],"mappings":";;;;;;AAAA,mFAAqG;AACrG,6IAA0I;AAC1I,mFAAkF;AAClF,yHAA6E;AAY7E,MAAM,gBAAgB,GAA4C,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzH,MAAsB,0BAA0B;IAAhD;QAGqB,sBAAiB,GAAG,IAAI,uCAAmB,CACxD,2BAA2B,EAC3B,gCAAsB,CAAC,iBAAiB,EAAE,CAC7C,CAAC;QACe,oBAAe,GAAG,IAAI,uCAAmB,CAAC,yBAAyB,EAAE,gCAAsB,CAAC,GAAG,EAAE,CAAC,CAAC;IAwCxH,CAAC;IApCU,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,OAAa,IAAI,IAAI,EAAE;QACrD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC7C,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,aAAa,CAAC,GAAW;QAC7B,OAAO,IAAI,wBAAU,CACjB,wBAAwB,EACxB,IAAI,qDAAyB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,EAC3E,IAAI,kCAAoB,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,EACtD,IAAI,CAAC,iBAAiB,CACzB,CAAC;IACN,CAAC;IAEO,WAAW,CAAC,MAAc;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,YAAY,CAAC,MAAmB,EAAE,QAAgC;QACtE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW;YAC3C,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW;YAC3C,EAAE,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAW;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAgB;YACxC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAW;YACrC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW;SAC9C,CAAC,CAAC,CAAC;IACR,CAAC;CACJ;AA/CD,gEA+CC"}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { ISimpleConfig } from "@golemio/core/dist/helpers/configuration/ISimpleConfig";
|
|
2
|
-
import { AbstractOpendataDatasource } from "./AbstractOpendataDatasource";
|
|
3
|
-
export default class OpendataDatasource extends AbstractOpendataDatasource {
|
|
4
|
-
name: string;
|
|
5
|
-
private readonly baseUrl;
|
|
6
|
-
constructor(config: ISimpleConfig);
|
|
7
|
-
protected buildUrl(wsi: string, date: Date): string;
|
|
8
|
-
}
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
-
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
16
|
-
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
17
|
-
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
18
|
-
const AbstractOpendataDatasource_1 = require("./AbstractOpendataDatasource");
|
|
19
|
-
let OpendataDatasource = class OpendataDatasource extends AbstractOpendataDatasource_1.AbstractOpendataDatasource {
|
|
20
|
-
constructor(config) {
|
|
21
|
-
super();
|
|
22
|
-
this.name = "OpendataDatasource";
|
|
23
|
-
this.baseUrl = config.getValue("module.chmu.opendata.baseUrl");
|
|
24
|
-
}
|
|
25
|
-
buildUrl(wsi, date) {
|
|
26
|
-
const dt = luxon_1.DateTime.fromJSDate(date).setZone("Europe/Prague");
|
|
27
|
-
const yyyymmdd = dt.toFormat("yyyyMMdd");
|
|
28
|
-
return `${this.baseUrl}/10m-${wsi}-${yyyymmdd}.json`;
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
OpendataDatasource = __decorate([
|
|
32
|
-
(0, tsyringe_1.injectable)(),
|
|
33
|
-
__param(0, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.SimpleConfig)),
|
|
34
|
-
__metadata("design:paramtypes", [Object])
|
|
35
|
-
], OpendataDatasource);
|
|
36
|
-
exports.default = OpendataDatasource;
|
|
37
|
-
//# sourceMappingURL=OpendataDatasource.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"OpendataDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/OpendataDatasource.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AACA,wEAAqE;AACrE,2DAA2D;AAC3D,iEAAwE;AACxE,6EAA0E;AAG3D,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,uDAA0B;IAKtE,YAA4C,MAAqB;QAC7D,KAAK,EAAE,CAAC;QALL,SAAI,GAAG,oBAAoB,CAAC;QAM/B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC;IACnE,CAAC;IAES,QAAQ,CAAC,GAAW,EAAE,IAAU;QACtC,MAAM,EAAE,GAAG,gBAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACzC,OAAO,GAAG,IAAI,CAAC,OAAO,QAAQ,GAAG,IAAI,QAAQ,OAAO,CAAC;IACzD,CAAC;CACJ,CAAA;AAfoB,kBAAkB;IADtC,IAAA,qBAAU,GAAE;IAMI,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,YAAY,CAAC,CAAA;;GAL1B,kBAAkB,CAetC;kBAfoB,kBAAkB"}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { ISimpleConfig } from "@golemio/core/dist/helpers/configuration/ISimpleConfig";
|
|
2
|
-
import { AbstractOpendataDatasource } from "./AbstractOpendataDatasource";
|
|
3
|
-
export default class OpendataHistoricalDatasource extends AbstractOpendataDatasource {
|
|
4
|
-
name: string;
|
|
5
|
-
private readonly baseUrl;
|
|
6
|
-
constructor(config: ISimpleConfig);
|
|
7
|
-
protected buildUrl(wsi: string, date: Date): string;
|
|
8
|
-
}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
-
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
-
};
|
|
11
|
-
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
-
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
|
|
16
|
-
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
17
|
-
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
18
|
-
const AbstractOpendataDatasource_1 = require("./AbstractOpendataDatasource");
|
|
19
|
-
let OpendataHistoricalDatasource = class OpendataHistoricalDatasource extends AbstractOpendataDatasource_1.AbstractOpendataDatasource {
|
|
20
|
-
constructor(config) {
|
|
21
|
-
super();
|
|
22
|
-
this.name = "OpendataHistoricalDatasource";
|
|
23
|
-
this.baseUrl = config.getValue("module.chmu.opendata.historicalBaseUrl");
|
|
24
|
-
}
|
|
25
|
-
buildUrl(wsi, date) {
|
|
26
|
-
const dt = luxon_1.DateTime.fromJSDate(date, { zone: "utc" });
|
|
27
|
-
const year = dt.toFormat("yyyy");
|
|
28
|
-
const yearMonth = dt.toFormat("yyyyMM");
|
|
29
|
-
return `${this.baseUrl}/${year}/10m-${wsi}-${yearMonth}.json`;
|
|
30
|
-
}
|
|
31
|
-
};
|
|
32
|
-
OpendataHistoricalDatasource = __decorate([
|
|
33
|
-
(0, tsyringe_1.injectable)(),
|
|
34
|
-
__param(0, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.SimpleConfig)),
|
|
35
|
-
__metadata("design:paramtypes", [Object])
|
|
36
|
-
], OpendataHistoricalDatasource);
|
|
37
|
-
exports.default = OpendataHistoricalDatasource;
|
|
38
|
-
//# sourceMappingURL=OpendataHistoricalDatasource.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"OpendataHistoricalDatasource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/OpendataHistoricalDatasource.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AACA,wEAAqE;AACrE,2DAA2D;AAC3D,iEAAwE;AACxE,6EAA0E;AAG3D,IAAM,4BAA4B,GAAlC,MAAM,4BAA6B,SAAQ,uDAA0B;IAKhF,YAA4C,MAAqB;QAC7D,KAAK,EAAE,CAAC;QALL,SAAI,GAAG,8BAA8B,CAAC;QAMzC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,wCAAwC,CAAC,CAAC;IAC7E,CAAC;IAES,QAAQ,CAAC,GAAW,EAAE,IAAU;QACtC,MAAM,EAAE,GAAG,gBAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,QAAQ,GAAG,IAAI,SAAS,OAAO,CAAC;IAClE,CAAC;CACJ,CAAA;AAhBoB,4BAA4B;IADhD,IAAA,qBAAU,GAAE;IAMI,WAAA,IAAA,iBAAM,EAAC,qBAAS,CAAC,YAAY,CAAC,CAAA;;GAL1B,4BAA4B,CAgBhD;kBAhBoB,4BAA4B"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import IInputOpendataRecord from "../../schema-definitions/datasources/interfaces/IInputOpendataRecord";
|
|
2
|
-
import { IMeasurementDto } from "../../schema-definitions/definitions/interfaces/IMeasurementDto";
|
|
3
|
-
import { ITransformation } from "@golemio/core/dist/integration-engine";
|
|
4
|
-
import { AbstractTransformation } from "@golemio/core/dist/helpers/transformation/AbstractTransformation";
|
|
5
|
-
export declare class OpendataMeasurementsTransformation extends AbstractTransformation<IInputOpendataRecord[], IMeasurementDto | null> implements ITransformation {
|
|
6
|
-
name: string;
|
|
7
|
-
/**
|
|
8
|
-
* Pivots long-format opendata records into wide IMeasurementDto rows.
|
|
9
|
-
* Groups by (station_id, DT) and maps each ELEMENT to its DB column.
|
|
10
|
-
* DT is UTC ISO 8601; year/month/day/time are expressed in Europe/Prague local time.
|
|
11
|
-
*/
|
|
12
|
-
transform(data: IInputOpendataRecord[]): Promise<IMeasurementDto[]>;
|
|
13
|
-
protected transformInternal: (records: IInputOpendataRecord[]) => IMeasurementDto | null;
|
|
14
|
-
private groupByStationAndDt;
|
|
15
|
-
}
|
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
-
};
|
|
8
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.OpendataMeasurementsTransformation = void 0;
|
|
10
|
-
const AbstractTransformation_1 = require("@golemio/core/dist/helpers/transformation/AbstractTransformation");
|
|
11
|
-
const luxon_1 = require("@golemio/core/dist/shared/luxon");
|
|
12
|
-
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
13
|
-
/**
|
|
14
|
-
* Maps ČHMÚ opendata ELEMENT codes to IMeasurementDto columns.
|
|
15
|
-
* Optional transform function applies per-element unit conversion.
|
|
16
|
-
*/
|
|
17
|
-
const ELEMENT_COLUMN = {
|
|
18
|
-
T: { column: "air_temp" },
|
|
19
|
-
H: { column: "air_hum" },
|
|
20
|
-
P: { column: "air_pressure" },
|
|
21
|
-
F: { column: "wind_speed" },
|
|
22
|
-
D: { column: "wind_dir" },
|
|
23
|
-
SRA10M: { column: "precip" },
|
|
24
|
-
// SSV10M: sunshine duration in seconds over the last 10 minutes (max 600 s).
|
|
25
|
-
// DB stores sunlight in tenths of an hour (1 tenth = 6 min = 360 s), same unit as FTP SSV1H.
|
|
26
|
-
// Max per 10-min interval: 600 s / 360 ≈ 1.67 tenths.
|
|
27
|
-
SSV10M: { column: "sunlight", transform: (v) => v / 360 },
|
|
28
|
-
TMA: { column: "air_temp_max" },
|
|
29
|
-
TMI: { column: "air_temp_min" },
|
|
30
|
-
TPM: { column: "air_temp_ground" },
|
|
31
|
-
Fprum: { column: "wind_speed_avg" },
|
|
32
|
-
Dprum: { column: "wind_dir_avg" },
|
|
33
|
-
Fmax: { column: "wind_gust" },
|
|
34
|
-
Dmax: { column: "wind_gust_dir" },
|
|
35
|
-
Casmax: { column: "wind_gust_time" },
|
|
36
|
-
SCEa: { column: "snow_depth" },
|
|
37
|
-
SVHa: { column: "snow_water" },
|
|
38
|
-
RGLB10: { column: "radiation_glob" },
|
|
39
|
-
REUV10: { column: "radiation_uv" },
|
|
40
|
-
REUV25: { column: "radiation_uv" },
|
|
41
|
-
T05: { column: "soil_temp_5cm" },
|
|
42
|
-
T10: { column: "soil_temp_10cm" },
|
|
43
|
-
T20: { column: "soil_temp_20cm" },
|
|
44
|
-
T50: { column: "soil_temp_50cm" },
|
|
45
|
-
T100: { column: "soil_temp_100cm" },
|
|
46
|
-
};
|
|
47
|
-
let OpendataMeasurementsTransformation = class OpendataMeasurementsTransformation extends AbstractTransformation_1.AbstractTransformation {
|
|
48
|
-
constructor() {
|
|
49
|
-
super(...arguments);
|
|
50
|
-
this.name = "OpendataMeasurementsTransformation";
|
|
51
|
-
this.transformInternal = (records) => {
|
|
52
|
-
const { station_id, DT } = records[0]; // group key — same across all records
|
|
53
|
-
const dt = luxon_1.DateTime.fromISO(DT).setZone("Europe/Prague");
|
|
54
|
-
if (!dt.isValid) {
|
|
55
|
-
return null;
|
|
56
|
-
}
|
|
57
|
-
const dto = {
|
|
58
|
-
measured_at: dt.toUTC().toISO(),
|
|
59
|
-
station_id,
|
|
60
|
-
year: dt.year,
|
|
61
|
-
month: dt.month,
|
|
62
|
-
day: dt.day,
|
|
63
|
-
time: dt.toFormat("HH:mm"),
|
|
64
|
-
air_temp: null,
|
|
65
|
-
air_hum: null,
|
|
66
|
-
precip: null,
|
|
67
|
-
air_pressure: null,
|
|
68
|
-
wind_speed: null,
|
|
69
|
-
wind_dir: null,
|
|
70
|
-
sunlight: null,
|
|
71
|
-
soil_hum_0_10: null,
|
|
72
|
-
soil_hum_10_50: null,
|
|
73
|
-
soil_hum_50_100: null,
|
|
74
|
-
air_temp_max: null,
|
|
75
|
-
air_temp_min: null,
|
|
76
|
-
air_temp_ground: null,
|
|
77
|
-
wind_speed_avg: null,
|
|
78
|
-
wind_dir_avg: null,
|
|
79
|
-
wind_gust: null,
|
|
80
|
-
wind_gust_dir: null,
|
|
81
|
-
wind_gust_time: null,
|
|
82
|
-
snow_depth: null,
|
|
83
|
-
snow_water: null,
|
|
84
|
-
radiation_glob: null,
|
|
85
|
-
radiation_uv: null,
|
|
86
|
-
soil_temp_5cm: null,
|
|
87
|
-
soil_temp_10cm: null,
|
|
88
|
-
soil_temp_20cm: null,
|
|
89
|
-
soil_temp_50cm: null,
|
|
90
|
-
soil_temp_100cm: null,
|
|
91
|
-
data_source: "opendata",
|
|
92
|
-
};
|
|
93
|
-
for (const record of records) {
|
|
94
|
-
const mapping = ELEMENT_COLUMN[record.ELEMENT];
|
|
95
|
-
if (!mapping || record.VAL === "") {
|
|
96
|
-
continue;
|
|
97
|
-
}
|
|
98
|
-
dto[mapping.column] = mapping.transform ? mapping.transform(record.VAL) : record.VAL;
|
|
99
|
-
}
|
|
100
|
-
return dto;
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* Pivots long-format opendata records into wide IMeasurementDto rows.
|
|
105
|
-
* Groups by (station_id, DT) and maps each ELEMENT to its DB column.
|
|
106
|
-
* DT is UTC ISO 8601; year/month/day/time are expressed in Europe/Prague local time.
|
|
107
|
-
*/
|
|
108
|
-
async transform(data) {
|
|
109
|
-
const groups = this.groupByStationAndDt(data);
|
|
110
|
-
return this.transformArray(groups).filter((dto) => dto !== null);
|
|
111
|
-
}
|
|
112
|
-
groupByStationAndDt(data) {
|
|
113
|
-
const groups = new Map();
|
|
114
|
-
for (const record of data) {
|
|
115
|
-
const key = `${record.station_id}|${record.DT}`;
|
|
116
|
-
if (!groups.has(key)) {
|
|
117
|
-
groups.set(key, []);
|
|
118
|
-
}
|
|
119
|
-
groups.get(key).push(record);
|
|
120
|
-
}
|
|
121
|
-
return Array.from(groups.values());
|
|
122
|
-
}
|
|
123
|
-
};
|
|
124
|
-
exports.OpendataMeasurementsTransformation = OpendataMeasurementsTransformation;
|
|
125
|
-
exports.OpendataMeasurementsTransformation = OpendataMeasurementsTransformation = __decorate([
|
|
126
|
-
(0, tsyringe_1.injectable)()
|
|
127
|
-
], OpendataMeasurementsTransformation);
|
|
128
|
-
//# sourceMappingURL=OpendataMeasurementsTransformation.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"OpendataMeasurementsTransformation.js","sourceRoot":"","sources":["../../../src/integration-engine/transformations/OpendataMeasurementsTransformation.ts"],"names":[],"mappings":";;;;;;;;;AAGA,6GAA0G;AAC1G,2DAA2D;AAC3D,iEAAgE;AAahE;;;GAGG;AACH,MAAM,cAAc,GAA6C;IAC7D,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;IACzB,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;IACxB,CAAC,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;IAC7B,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE;IAC3B,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;IACzB,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE;IAC5B,6EAA6E;IAC7E,6FAA6F;IAC7F,sDAAsD;IACtD,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,EAAE;IACzD,GAAG,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;IAC/B,GAAG,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;IAC/B,GAAG,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE;IAClC,KAAK,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE;IACnC,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;IACjC,IAAI,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;IAC7B,IAAI,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;IACjC,MAAM,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE;IACpC,IAAI,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE;IAC9B,IAAI,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE;IAC9B,MAAM,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE;IACpC,MAAM,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;IAClC,MAAM,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;IAClC,GAAG,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;IAChC,GAAG,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE;IACjC,GAAG,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE;IACjC,GAAG,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE;IACjC,IAAI,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE;CACtC,CAAC;AAGK,IAAM,kCAAkC,GAAxC,MAAM,kCACT,SAAQ,+CAAsE;IAD3E;;QAII,SAAI,GAAG,oCAAoC,CAAC;QAYzC,sBAAiB,GAAG,CAAC,OAA+B,EAA0B,EAAE;YACtF,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,sCAAsC;YAC7E,MAAM,EAAE,GAAG,gBAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;YACzD,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,MAAM,GAAG,GAAoB;gBACzB,WAAW,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,EAAG;gBAChC,UAAU;gBACV,IAAI,EAAE,EAAE,CAAC,IAAI;gBACb,KAAK,EAAE,EAAE,CAAC,KAAK;gBACf,GAAG,EAAE,EAAE,CAAC,GAAG;gBACX,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAC1B,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,IAAI;gBAClB,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,IAAI;gBACd,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,IAAI;gBAClB,YAAY,EAAE,IAAI;gBAClB,eAAe,EAAE,IAAI;gBACrB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,IAAI;gBAChB,UAAU,EAAE,IAAI;gBAChB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI;gBACpB,cAAc,EAAE,IAAI;gBACpB,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,IAAI;gBACrB,WAAW,EAAE,UAAU;aAC1B,CAAC;YAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC/C,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,CAAC;oBAChC,SAAS;gBACb,CAAC;gBACD,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;YACzF,CAAC;YAED,OAAO,GAAG,CAAC;QACf,CAAC,CAAC;IAaN,CAAC;IA5EG;;;;OAIG;IACI,KAAK,CAAC,SAAS,CAAC,IAA4B;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAA0B,EAAE,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC;IAC7F,CAAC;IAyDO,mBAAmB,CAAC,IAA4B;QACpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkC,CAAC;QACzD,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;CACJ,CAAA;AAlFY,gFAAkC;6CAAlC,kCAAkC;IAD9C,IAAA,qBAAU,GAAE;GACA,kCAAkC,CAkF9C"}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export default interface IDownloadOpendataHistoricalMessage {
|
|
2
|
-
/** ISO 8601 timestamp - inclusive start of the range to download. Required; no default. */
|
|
3
|
-
from: string;
|
|
4
|
-
/** ISO 8601 timestamp - exclusive end of the range to download. Required; no default. */
|
|
5
|
-
to: string;
|
|
6
|
-
/** Station ID from chmu.stations. When set together with wsi, task runs in single-station mode. */
|
|
7
|
-
station_id?: string;
|
|
8
|
-
/** WSI identifier of the station. When set together with station_id, task runs in single-station mode. */
|
|
9
|
-
wsi?: string;
|
|
10
|
-
}
|
package/dist/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"IDownloadOpendataHistoricalMessage.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/interfaces/IDownloadOpendataHistoricalMessage.ts"],"names":[],"mappings":""}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
export default interface IDownloadOpendataMessage {
|
|
2
|
-
/** ISO 8601 date to fetch opendata for. Defaults to today when omitted. */
|
|
3
|
-
date?: string;
|
|
4
|
-
/** WSI identifier of the station. When set together with station_id, task runs in single-station mode. */
|
|
5
|
-
wsi?: string;
|
|
6
|
-
/** Station ID from chmu.stations. Required when wsi is set. */
|
|
7
|
-
station_id?: string;
|
|
8
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"IDownloadOpendataMessage.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/interfaces/IDownloadOpendataMessage.ts"],"names":[],"mappings":""}
|
package/dist/integration-engine/workers/tasks/DownloadOpendataHistoricalMeasurementsTask.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { StationsRepository } from "../../data-access/StationsRepository";
|
|
2
|
-
import { MeasurementsRepository } from "../../data-access/MeasurementsRepository";
|
|
3
|
-
import OpendataHistoricalDatasource from "../../datasources/OpendataHistoricalDatasource";
|
|
4
|
-
import { OpendataMeasurementsTransformation } from "../../transformations/OpendataMeasurementsTransformation";
|
|
5
|
-
import { ILogger } from "@golemio/core/dist/helpers/logger/LoggerProvider";
|
|
6
|
-
import { AbstractTask } from "@golemio/core/dist/integration-engine";
|
|
7
|
-
import IDownloadOpendataHistoricalMessage from "../interfaces/IDownloadOpendataHistoricalMessage";
|
|
8
|
-
import { DownloadOpendataHistoricalMessageSchema } from "./DownloadOpendataHistoricalMessageSchema";
|
|
9
|
-
export default class DownloadOpendataHistoricalMeasurementsTask extends AbstractTask<IDownloadOpendataHistoricalMessage> {
|
|
10
|
-
private stationsRepository;
|
|
11
|
-
private datasource;
|
|
12
|
-
private transformation;
|
|
13
|
-
private repository;
|
|
14
|
-
private logger;
|
|
15
|
-
queueName: string;
|
|
16
|
-
protected schema: typeof DownloadOpendataHistoricalMessageSchema;
|
|
17
|
-
constructor(stationsRepository: StationsRepository, datasource: OpendataHistoricalDatasource, transformation: OpendataMeasurementsTransformation, repository: MeasurementsRepository, logger: ILogger);
|
|
18
|
-
protected execute: (data: IDownloadOpendataHistoricalMessage) => Promise<void>;
|
|
19
|
-
private generateSubtasks;
|
|
20
|
-
private processStation;
|
|
21
|
-
private generateStationMonthSubtasks;
|
|
22
|
-
private processStationMeasurements;
|
|
23
|
-
private validateRange;
|
|
24
|
-
/** Splits [from, to) into calendar-month intervals (UTC, matching DT's format); clips the first/last interval. */
|
|
25
|
-
private static generateMonthIntervals;
|
|
26
|
-
}
|