@golemio/traffic-common 1.0.1-dev.1040804803 → 1.0.1-dev.1047251822
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/postgresql/20231023081724-replace-rsd-tmc-osm-mapping-data.js +53 -0
- package/db/migrations/postgresql/sqls/20231023081724-replace-rsd-tmc-osm-mapping-data-down.sql +2 -0
- package/db/migrations/postgresql/sqls/20231023081724-replace-rsd-tmc-osm-mapping-data-up.sql +28 -0
- package/dist/integration-engine/repositories/RsdTmcOsmMappingRepository.d.ts +20 -2
- package/dist/integration-engine/repositories/RsdTmcOsmMappingRepository.js +42 -5
- package/dist/integration-engine/repositories/RsdTmcOsmMappingRepository.js.map +1 -1
- package/dist/integration-engine/workers/tasks/RefreshRsdTmcOsmMappingDataTask.js +2 -8
- package/dist/integration-engine/workers/tasks/RefreshRsdTmcOsmMappingDataTask.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var dbm;
|
|
4
|
+
var type;
|
|
5
|
+
var seed;
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var Promise;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* We receive the dbmigrate dependency from dbmigrate initially.
|
|
12
|
+
* This enables us to not have to rely on NODE_PATH.
|
|
13
|
+
*/
|
|
14
|
+
exports.setup = function(options, seedLink) {
|
|
15
|
+
dbm = options.dbmigrate;
|
|
16
|
+
type = dbm.dataType;
|
|
17
|
+
seed = seedLink;
|
|
18
|
+
Promise = options.Promise;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
exports.up = function(db) {
|
|
22
|
+
var filePath = path.join(__dirname, 'sqls', '20231023081724-replace-rsd-tmc-osm-mapping-data-up.sql');
|
|
23
|
+
return new Promise( function( resolve, reject ) {
|
|
24
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
25
|
+
if (err) return reject(err);
|
|
26
|
+
console.log('received data: ' + data);
|
|
27
|
+
|
|
28
|
+
resolve(data);
|
|
29
|
+
});
|
|
30
|
+
})
|
|
31
|
+
.then(function(data) {
|
|
32
|
+
return db.runSql(data);
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
exports.down = function(db) {
|
|
37
|
+
var filePath = path.join(__dirname, 'sqls', '20231023081724-replace-rsd-tmc-osm-mapping-data-down.sql');
|
|
38
|
+
return new Promise( function( resolve, reject ) {
|
|
39
|
+
fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
|
|
40
|
+
if (err) return reject(err);
|
|
41
|
+
console.log('received data: ' + data);
|
|
42
|
+
|
|
43
|
+
resolve(data);
|
|
44
|
+
});
|
|
45
|
+
})
|
|
46
|
+
.then(function(data) {
|
|
47
|
+
return db.runSql(data);
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
exports._meta = {
|
|
52
|
+
"version": 1
|
|
53
|
+
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
CREATE PROCEDURE create_tmp_table(IN tableName Text, OUT tableNameTmp Text)
|
|
2
|
+
LANGUAGE plpgsql
|
|
3
|
+
SET search_path FROM CURRENT
|
|
4
|
+
AS $PROCEDURE$
|
|
5
|
+
BEGIN
|
|
6
|
+
SELECT concat(tableName, '_tmp') INTO tableNameTmp;
|
|
7
|
+
EXECUTE format('DROP TABLE IF EXISTS %I;', tableNameTmp);
|
|
8
|
+
EXECUTE format('CREATE TABLE %I (LIKE %I INCLUDING ALL);', tableNameTmp, tableName);
|
|
9
|
+
END;
|
|
10
|
+
$PROCEDURE$;
|
|
11
|
+
|
|
12
|
+
CREATE PROCEDURE replace_with_tmp_table(IN tableName Text)
|
|
13
|
+
LANGUAGE plpgsql
|
|
14
|
+
SET search_path FROM CURRENT
|
|
15
|
+
AS $PROCEDURE$
|
|
16
|
+
DECLARE
|
|
17
|
+
tableNameOld Text;
|
|
18
|
+
tableNameTmp Text;
|
|
19
|
+
BEGIN
|
|
20
|
+
SELECT concat(tableName, '_old') INTO tableNameOld;
|
|
21
|
+
SELECT concat(tableName, '_tmp') INTO tableNameTmp;
|
|
22
|
+
|
|
23
|
+
EXECUTE format('LOCK %I IN EXCLUSIVE MODE', tableName);
|
|
24
|
+
EXECUTE format('ALTER TABLE %I RENAME TO %I', tableName, tableNameOld);
|
|
25
|
+
EXECUTE format('ALTER TABLE %I RENAME TO %I', tableNameTmp, tableName);
|
|
26
|
+
EXECUTE format('DROP TABLE %I', tableNameOld);
|
|
27
|
+
END;
|
|
28
|
+
$PROCEDURE$;
|
|
@@ -1,12 +1,30 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine/models";
|
|
3
|
-
import IPostgresConnector from "@golemio/core/dist/integration-engine/connectors/interfaces/IPostgresConnector";
|
|
4
2
|
import { ILogger } from "@golemio/core/dist/helpers";
|
|
3
|
+
import IPostgresConnector from "@golemio/core/dist/integration-engine/connectors/interfaces/IPostgresConnector";
|
|
4
|
+
import { IModel, PostgresModel } from "@golemio/core/dist/integration-engine/models";
|
|
5
5
|
import { Readable } from "stream";
|
|
6
6
|
export declare class RsdTmcOsmMappingRepository extends PostgresModel implements IModel {
|
|
7
7
|
private connector;
|
|
8
8
|
private logger;
|
|
9
9
|
constructor(connector: IPostgresConnector, logger: ILogger);
|
|
10
10
|
cleanTable(): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Replace all current data with data from a given input stream
|
|
13
|
+
*
|
|
14
|
+
* @param data The input stream to insert from
|
|
15
|
+
*/
|
|
16
|
+
replaceWithStreamData(data: Readable): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Insert data from a given input stream
|
|
19
|
+
*
|
|
20
|
+
* @param data The input stream to insert from
|
|
21
|
+
*/
|
|
11
22
|
streamDataIn(data: Readable): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Insert data from a given input stream to a given table
|
|
25
|
+
*
|
|
26
|
+
* @param data The input stream to insert from
|
|
27
|
+
* @param tableName The name of the table to insert to
|
|
28
|
+
*/
|
|
29
|
+
private streamDataInTable;
|
|
12
30
|
}
|
|
@@ -22,12 +22,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
22
22
|
};
|
|
23
23
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
24
|
exports.RsdTmcOsmMappingRepository = void 0;
|
|
25
|
-
const
|
|
26
|
-
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
25
|
+
const _sch_1 = require("../../schema-definitions");
|
|
27
26
|
const RsdTmcOsmMappingModel_1 = require("../../schema-definitions/models/RsdTmcOsmMappingModel");
|
|
28
|
-
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
29
27
|
const ioc_1 = require("@golemio/core/dist/integration-engine/ioc");
|
|
30
|
-
const
|
|
28
|
+
const models_1 = require("@golemio/core/dist/integration-engine/models");
|
|
29
|
+
const golemio_validator_1 = require("@golemio/core/dist/shared/golemio-validator");
|
|
30
|
+
const sequelize_1 = require("@golemio/core/dist/shared/sequelize");
|
|
31
|
+
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
|
|
31
32
|
const pg_copy_streams_1 = require("pg-copy-streams");
|
|
32
33
|
let RsdTmcOsmMappingRepository = exports.RsdTmcOsmMappingRepository = class RsdTmcOsmMappingRepository extends models_1.PostgresModel {
|
|
33
34
|
constructor(connector, logger) {
|
|
@@ -51,14 +52,50 @@ let RsdTmcOsmMappingRepository = exports.RsdTmcOsmMappingRepository = class RsdT
|
|
|
51
52
|
});
|
|
52
53
|
});
|
|
53
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Replace all current data with data from a given input stream
|
|
57
|
+
*
|
|
58
|
+
* @param data The input stream to insert from
|
|
59
|
+
*/
|
|
60
|
+
replaceWithStreamData(data) {
|
|
61
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
62
|
+
const connection = this.connector.getConnection();
|
|
63
|
+
const res = yield connection.query(`CALL ${_sch_1.RsdTmcOsmMappingTable.pgSchema}.create_tmp_table($1, null)`, {
|
|
64
|
+
bind: [this.tableName],
|
|
65
|
+
plain: true,
|
|
66
|
+
type: sequelize_1.QueryTypes.SELECT,
|
|
67
|
+
});
|
|
68
|
+
if (typeof (res === null || res === void 0 ? void 0 : res.tablenametmp) === "undefined") {
|
|
69
|
+
throw new TypeError(`Got undefined tablenametmp from call of create_tmp_table for table ${this.tableName}.`);
|
|
70
|
+
}
|
|
71
|
+
yield this.streamDataInTable(data, res.tablenametmp);
|
|
72
|
+
yield connection.query(`CALL ${_sch_1.RsdTmcOsmMappingTable.pgSchema}.replace_with_tmp_table($1)`, { bind: [this.tableName] });
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Insert data from a given input stream
|
|
77
|
+
*
|
|
78
|
+
* @param data The input stream to insert from
|
|
79
|
+
*/
|
|
54
80
|
streamDataIn(data) {
|
|
81
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
82
|
+
return this.streamDataInTable(data, _sch_1.RsdTmcOsmMappingTable.pgTableName);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Insert data from a given input stream to a given table
|
|
87
|
+
*
|
|
88
|
+
* @param data The input stream to insert from
|
|
89
|
+
* @param tableName The name of the table to insert to
|
|
90
|
+
*/
|
|
91
|
+
streamDataInTable(data, tableName) {
|
|
55
92
|
return __awaiter(this, void 0, void 0, function* () {
|
|
56
93
|
const connection = this.connector.getConnection();
|
|
57
94
|
const client = yield connection.connectionManager.getConnection({ type: "write" });
|
|
58
95
|
// copy transformed data to tmp table by stream
|
|
59
96
|
yield new Promise((resolve, reject) => {
|
|
60
97
|
const stream = client
|
|
61
|
-
.query((0, pg_copy_streams_1.from)(`COPY "${_sch_1.RsdTmcOsmMappingTable.pgSchema}".${
|
|
98
|
+
.query((0, pg_copy_streams_1.from)(`COPY "${_sch_1.RsdTmcOsmMappingTable.pgSchema}".${tableName}
|
|
62
99
|
FROM STDIN DELIMITER ',' CSV HEADER;`))
|
|
63
100
|
.on("error", (err) => __awaiter(this, void 0, void 0, function* () {
|
|
64
101
|
this.logger.error(`Copying data error (${this.name}): ${err.toString()}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RsdTmcOsmMappingRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/repositories/RsdTmcOsmMappingRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"RsdTmcOsmMappingRepository.js","sourceRoot":"","sources":["../../../src/integration-engine/repositories/RsdTmcOsmMappingRepository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,mDAA6C;AAC7C,iGAA0E;AAG1E,mEAA2E;AAC3E,yEAAqF;AACrF,mFAAkF;AAClF,mEAAiE;AACjE,iEAAwE;AACxE,qDAAmD;AAI5C,IAAM,0BAA0B,wCAAhC,MAAM,0BAA2B,SAAQ,sBAAa;IACzD,YACsD,SAA6B,EACxC,MAAe;QAEtD,KAAK,CACD,4BAA4B,EAC5B;YACI,yBAAyB,EAAE,6CAAqB,CAAC,cAAc;YAC/D,WAAW,EAAE,4BAAqB,CAAC,WAAW;YAC9C,QAAQ,EAAE,4BAAqB,CAAC,QAAQ;YACxC,UAAU,EAAE,YAAY;YACxB,kBAAkB,EAAE,CAAC,IAAI,CAAC;YAC1B,2BAA2B,EAAE;gBACzB,UAAU,EAAE,KAAK;aACpB;SACJ,EACD,IAAI,uCAAmB,CAAC,2BAA2B,EAAE,6CAAqB,CAAC,UAAU,CAAC,CACzF,CAAC;QAhBgD,cAAS,GAAT,SAAS,CAAoB;QACxC,WAAM,GAAN,MAAM,CAAS;IAgB1D,CAAC;IAEY,UAAU;;YACnB,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;gBAC9B,QAAQ,EAAE,IAAI;aACjB,CAAC,CAAC;QACP,CAAC;KAAA;IAED;;;;OAIG;IACU,qBAAqB,CAAC,IAAc;;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;YAClD,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAC9B,QAAQ,4BAAqB,CAAC,QAAQ,6BAA6B,EACnE;gBACI,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC;gBACtB,KAAK,EAAE,IAAI;gBACX,IAAI,EAAE,sBAAU,CAAC,MAAM;aAC1B,CACJ,CAAC;YACF,IAAI,OAAO,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,YAAY,CAAA,KAAK,WAAW,EAAE;gBAC1C,MAAM,IAAI,SAAS,CAAC,sEAAsE,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;aAChH;YACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;YACrD,MAAM,UAAU,CAAC,KAAK,CAAC,QAAQ,4BAAqB,CAAC,QAAQ,6BAA6B,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5H,CAAC;KAAA;IAED;;;;OAIG;IACU,YAAY,CAAC,IAAc;;YACpC,OAAO,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,4BAAqB,CAAC,WAAW,CAAC,CAAC;QAC3E,CAAC;KAAA;IAED;;;;;OAKG;IACW,iBAAiB,CAAC,IAAc,EAAE,SAAiB;;YAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;YAClD,MAAM,MAAM,GAAQ,MAAM,UAAU,CAAC,iBAAiB,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAExF,+CAA+C;YAC/C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACxC,MAAM,MAAM,GAAG,MAAM;qBAChB,KAAK,CACF,IAAA,sBAAQ,EACJ,SAAS,4BAAqB,CAAC,QAAQ,KAAK,SAAS;6DAChB,CACxC,CACJ;qBACA,EAAE,CAAC,OAAO,EAAE,CAAO,GAAQ,EAAE,EAAE;oBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,IAAI,MAAM,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC1E,MAAM,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC7D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAA,CAAC;qBACD,EAAE,CAAC,QAAQ,EAAE,GAAS,EAAE;oBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;oBACrD,MAAM,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;oBAC7D,OAAO,OAAO,EAAE,CAAC;gBACrB,CAAC,CAAA,CAAC,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;CACJ,CAAA;qCA1FY,0BAA0B;IADtC,IAAA,qBAAU,GAAE;IAGJ,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,iBAAiB,CAAC,CAAA;IACxC,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;;GAHzB,0BAA0B,CA0FtC"}
|
|
@@ -23,12 +23,6 @@ class RefreshRsdTmcOsmMappingDataTask extends integration_engine_1.AbstractEmpty
|
|
|
23
23
|
execute() {
|
|
24
24
|
return __awaiter(this, void 0, void 0, function* () {
|
|
25
25
|
let dataStream;
|
|
26
|
-
try {
|
|
27
|
-
yield this.repository.cleanTable();
|
|
28
|
-
}
|
|
29
|
-
catch (err) {
|
|
30
|
-
throw new golemio_errors_1.GeneralError("Error while cleaning table.", this.constructor.name, err);
|
|
31
|
-
}
|
|
32
26
|
try {
|
|
33
27
|
dataStream = yield this.streamedStrategy.getRawData();
|
|
34
28
|
}
|
|
@@ -36,10 +30,10 @@ class RefreshRsdTmcOsmMappingDataTask extends integration_engine_1.AbstractEmpty
|
|
|
36
30
|
throw new golemio_errors_1.GeneralError("Error while getting data stream.", this.constructor.name, err);
|
|
37
31
|
}
|
|
38
32
|
try {
|
|
39
|
-
yield this.repository.
|
|
33
|
+
yield this.repository.replaceWithStreamData(dataStream.data);
|
|
40
34
|
}
|
|
41
35
|
catch (err) {
|
|
42
|
-
throw new golemio_errors_1.GeneralError("Error while
|
|
36
|
+
throw new golemio_errors_1.GeneralError("Error while replacing old data with new data from stream.", this.constructor.name, err);
|
|
43
37
|
}
|
|
44
38
|
});
|
|
45
39
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RefreshRsdTmcOsmMappingDataTask.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/tasks/RefreshRsdTmcOsmMappingDataTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8EAAwG;AAExG,6EAAwE;AAGxE,MAAa,+BAAgC,SAAQ,sCAAiB;IAIlE,YACI,WAAmB,EACX,gBAA8C,EAC9C,UAAsC;QAE9C,KAAK,CAAC,WAAW,CAAC,CAAC;QAHX,qBAAgB,GAAhB,gBAAgB,CAA8B;QAC9C,eAAU,GAAV,UAAU,CAA4B;QANlC,cAAS,GAAG,yBAAyB,CAAC;QACtC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;IAQxD,CAAC;IAEe,OAAO;;YACnB,IAAI,UAA8B,CAAC;YAEnC,IAAI;gBACA,
|
|
1
|
+
{"version":3,"file":"RefreshRsdTmcOsmMappingDataTask.js","sourceRoot":"","sources":["../../../../src/integration-engine/workers/tasks/RefreshRsdTmcOsmMappingDataTask.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8EAAwG;AAExG,6EAAwE;AAGxE,MAAa,+BAAgC,SAAQ,sCAAiB;IAIlE,YACI,WAAmB,EACX,gBAA8C,EAC9C,UAAsC;QAE9C,KAAK,CAAC,WAAW,CAAC,CAAC;QAHX,qBAAgB,GAAhB,gBAAgB,CAA8B;QAC9C,eAAU,GAAV,UAAU,CAA4B;QANlC,cAAS,GAAG,yBAAyB,CAAC;QACtC,aAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;IAQxD,CAAC;IAEe,OAAO;;YACnB,IAAI,UAA8B,CAAC;YAEnC,IAAI;gBACA,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;aACzD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,6BAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC1F;YAED,IAAI;gBACA,MAAM,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAChE;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,6BAAY,CAAC,2DAA2D,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACnH;QACL,CAAC;KAAA;CACJ;AA3BD,0EA2BC"}
|