@golemio/core 4.0.0-dev.2672402710 → 4.0.0-rc.2674851974
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/data-access/postgres/DatabaseConnector.js +3 -10
- package/dist/helpers/data-access/postgres/DatabaseConnector.js.map +1 -1
- package/dist/helpers/data-access/postgres/IDatabaseInfo.d.ts +0 -1
- package/dist/integration-engine/datasources/index.d.ts +0 -5
- package/dist/integration-engine/datasources/index.js +0 -5
- package/dist/integration-engine/datasources/index.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.d.ts +1 -4
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js +1 -30
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.d.ts +0 -2
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +3 -42
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/ioc/Di.js +0 -1
- package/dist/integration-engine/ioc/Di.js.map +1 -1
- package/dist/output-gateway/ioc/Di.js +0 -1
- package/dist/output-gateway/ioc/Di.js.map +1 -1
- package/package.json +4 -6
- package/dist/integration-engine/datasources/BatchWritable.d.ts +0 -14
- package/dist/integration-engine/datasources/BatchWritable.js +0 -42
- package/dist/integration-engine/datasources/BatchWritable.js.map +0 -1
- package/dist/integration-engine/datasources/DataSourceStreamedPipeline.d.ts +0 -24
- package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js +0 -72
- package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js.map +0 -1
- package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.d.ts +0 -4
- package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js +0 -12
- package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js.map +0 -1
- package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.d.ts +0 -4
- package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js +0 -3
- package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map +0 -1
|
@@ -59,17 +59,10 @@ class DatabaseConnector {
|
|
|
59
59
|
};
|
|
60
60
|
}
|
|
61
61
|
getOptions(additionalOptions) {
|
|
62
|
-
const dialectOptions = {
|
|
63
|
-
application_name: `${this.databaseInfo.applicationName} (${os_1.default.hostname()})`,
|
|
64
|
-
};
|
|
65
|
-
if (this.databaseInfo.sslCaBase64) {
|
|
66
|
-
dialectOptions.ssl = {
|
|
67
|
-
ca: Buffer.from(this.databaseInfo.sslCaBase64, "base64").toString("utf-8"),
|
|
68
|
-
rejectUnauthorized: true,
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
62
|
return {
|
|
72
|
-
dialectOptions
|
|
63
|
+
dialectOptions: {
|
|
64
|
+
application_name: `${this.databaseInfo.applicationName} (${os_1.default.hostname()})`,
|
|
65
|
+
},
|
|
73
66
|
logging: this.log.silly,
|
|
74
67
|
pool: {
|
|
75
68
|
acquire: 60000,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DatabaseConnector.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/postgres/DatabaseConnector.ts"],"names":[],"mappings":";;;;;;AACA,4CAA6C;AAC7C,4CAAoB;AACpB,yCAA+C;AAK/C,MAAa,iBAAiB;IAG1B,YACqB,YAA2B,EAC3B,GAAY,EACZ,gBAA0B;QAF1B,iBAAY,GAAZ,YAAY,CAAe;QAC3B,QAAG,GAAH,GAAG,CAAS;QACZ,qBAAgB,GAAhB,gBAAgB,CAAU;QAGxC,YAAO,GAAG,KAAK,EAAE,iBAA2B,EAAsB,EAAE;YACvE,IAAI,CAAC;gBACD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,OAAO,IAAI,CAAC,UAAU,CAAC;gBAC3B,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;oBACtC,MAAM,IAAI,mBAAU,CAAC,qDAAqD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvG,CAAC;gBAED,IAAI,CAAC,UAAU,GAAG,IAAI,qBAAS,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAExG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;oBACnB,MAAM,IAAI,mBAAU,CAAC,0BAA0B,CAAC,CAAC;gBACrD,CAAC;gBAED,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,OAAO,IAAI,CAAC,UAAU,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,mBAAU,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC9F,CAAC;QACL,CAAC,CAAC;QAEK,kBAAa,GAAG,GAAc,EAAE;YACnC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnB,MAAM,IAAI,mBAAU,CAAC,mEAAmE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACrH,CAAC;YAED,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAC;QAEK,gBAAW,GAAG,KAAK,IAAsB,EAAE;YAC9C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,UAAW,CAAC,YAAY,EAAE,CAAC;gBAEtC,OAAO,IAAI,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oDAAoD,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC,CAAC;QAEK,eAAU,GAAG,KAAK,IAAmB,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAClC,CAAC;QACL,CAAC,CAAC;QAEK,YAAO,GAAG,KAAK,IAAmB,EAAE;YACvC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC,CAAC;IAvDC,CAAC;IAyDI,UAAU,CAAC,iBAA2B;QAC1C,
|
|
1
|
+
{"version":3,"file":"DatabaseConnector.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/postgres/DatabaseConnector.ts"],"names":[],"mappings":";;;;;;AACA,4CAA6C;AAC7C,4CAAoB;AACpB,yCAA+C;AAK/C,MAAa,iBAAiB;IAG1B,YACqB,YAA2B,EAC3B,GAAY,EACZ,gBAA0B;QAF1B,iBAAY,GAAZ,YAAY,CAAe;QAC3B,QAAG,GAAH,GAAG,CAAS;QACZ,qBAAgB,GAAhB,gBAAgB,CAAU;QAGxC,YAAO,GAAG,KAAK,EAAE,iBAA2B,EAAsB,EAAE;YACvE,IAAI,CAAC;gBACD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,OAAO,IAAI,CAAC,UAAU,CAAC;gBAC3B,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;oBACtC,MAAM,IAAI,mBAAU,CAAC,qDAAqD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvG,CAAC;gBAED,IAAI,CAAC,UAAU,GAAG,IAAI,qBAAS,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAExG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;oBACnB,MAAM,IAAI,mBAAU,CAAC,0BAA0B,CAAC,CAAC;gBACrD,CAAC;gBAED,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,OAAO,IAAI,CAAC,UAAU,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,mBAAU,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC9F,CAAC;QACL,CAAC,CAAC;QAEK,kBAAa,GAAG,GAAc,EAAE;YACnC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnB,MAAM,IAAI,mBAAU,CAAC,mEAAmE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACrH,CAAC;YAED,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAC;QAEK,gBAAW,GAAG,KAAK,IAAsB,EAAE;YAC9C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,UAAW,CAAC,YAAY,EAAE,CAAC;gBAEtC,OAAO,IAAI,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oDAAoD,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC,CAAC;QAEK,eAAU,GAAG,KAAK,IAAmB,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAClC,CAAC;QACL,CAAC,CAAC;QAEK,YAAO,GAAG,KAAK,IAAmB,EAAE;YACvC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC,CAAC;IAvDC,CAAC;IAyDI,UAAU,CAAC,iBAA2B;QAC1C,OAAO;YACH,cAAc,EAAE;gBACZ,gBAAgB,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,eAAe,KAAK,YAAE,CAAC,QAAQ,EAAE,GAAG;aAC9E;YACD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK;YACvB,IAAI,EAAE;gBACF,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC;gBAC/C,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBACjD,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;aACpD;YACD,KAAK,EAAE;gBACH,KAAK,EAAE;oBACH,0BAA0B;oBAC1B,iCAAiC;oBACjC,4BAA4B;oBAC5B,gCAAgC;oBAChC,iCAAiC;oBACjC,kCAAkC;oBAClC,cAAc;iBACjB;gBACD,GAAG,EAAE,CAAC;aACT;YACD,GAAG,IAAI,CAAC,gBAAgB;YACxB,GAAG,iBAAiB;SACvB,CAAC;IACN,CAAC;CACJ;AA5FD,8CA4FC"}
|
|
@@ -1,22 +1,17 @@
|
|
|
1
|
-
export * from "./BatchWritable";
|
|
2
1
|
export * from "./DataSource";
|
|
3
2
|
export * from "./DataSourceStream";
|
|
4
3
|
export * from "./DataSourceStreamManager";
|
|
5
4
|
export * from "./DataSourceStreamed";
|
|
6
|
-
export * from "./DataSourceStreamedPipeline";
|
|
7
5
|
export * from "./IDataSource";
|
|
8
6
|
export * from "./datatype-strategy/CSVDataTypeStrategy";
|
|
9
7
|
export * from "./datatype-strategy/IDataTypeStrategy";
|
|
10
8
|
export * from "./datatype-strategy/JSONDataTypeStrategy";
|
|
11
|
-
export * from "./datatype-strategy/JSONStreamTransformer";
|
|
12
9
|
export * from "./datatype-strategy/XMLDataTypeStrategy";
|
|
13
10
|
export * from "./protocol-strategy/FTPProtocolStrategy";
|
|
14
|
-
export * from "./protocol-strategy/HTTPRequestProtocolStrategyStreamed";
|
|
15
11
|
export * from "./protocol-strategy/GoogleCloudStorageProtocolStrategy";
|
|
16
12
|
export * from "./protocol-strategy/HTTPFetchProtocolStrategy";
|
|
17
13
|
export * from "./protocol-strategy/PaginatedHTTPProtocolStrategy";
|
|
18
14
|
export * from "./protocol-strategy/IProtocolStrategy";
|
|
19
|
-
export * from "./protocol-strategy/IStreamingProtocolStrategy";
|
|
20
15
|
export * from "./protocol-strategy/PostgresProtocolStrategy";
|
|
21
16
|
export * from "./protocol-strategy/PostgresProtocolStrategyStreamed";
|
|
22
17
|
export * from "./protocol-strategy/ProtocolStrategy";
|
|
@@ -15,25 +15,20 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
/* ie/datasources/index.ts */
|
|
18
|
-
__exportStar(require("./BatchWritable"), exports);
|
|
19
18
|
__exportStar(require("./DataSource"), exports);
|
|
20
19
|
__exportStar(require("./DataSourceStream"), exports);
|
|
21
20
|
__exportStar(require("./DataSourceStreamManager"), exports);
|
|
22
21
|
__exportStar(require("./DataSourceStreamed"), exports);
|
|
23
|
-
__exportStar(require("./DataSourceStreamedPipeline"), exports);
|
|
24
22
|
__exportStar(require("./IDataSource"), exports);
|
|
25
23
|
__exportStar(require("./datatype-strategy/CSVDataTypeStrategy"), exports);
|
|
26
24
|
__exportStar(require("./datatype-strategy/IDataTypeStrategy"), exports);
|
|
27
25
|
__exportStar(require("./datatype-strategy/JSONDataTypeStrategy"), exports);
|
|
28
|
-
__exportStar(require("./datatype-strategy/JSONStreamTransformer"), exports);
|
|
29
26
|
__exportStar(require("./datatype-strategy/XMLDataTypeStrategy"), exports);
|
|
30
27
|
__exportStar(require("./protocol-strategy/FTPProtocolStrategy"), exports);
|
|
31
|
-
__exportStar(require("./protocol-strategy/HTTPRequestProtocolStrategyStreamed"), exports);
|
|
32
28
|
__exportStar(require("./protocol-strategy/GoogleCloudStorageProtocolStrategy"), exports);
|
|
33
29
|
__exportStar(require("./protocol-strategy/HTTPFetchProtocolStrategy"), exports);
|
|
34
30
|
__exportStar(require("./protocol-strategy/PaginatedHTTPProtocolStrategy"), exports);
|
|
35
31
|
__exportStar(require("./protocol-strategy/IProtocolStrategy"), exports);
|
|
36
|
-
__exportStar(require("./protocol-strategy/IStreamingProtocolStrategy"), exports);
|
|
37
32
|
__exportStar(require("./protocol-strategy/PostgresProtocolStrategy"), exports);
|
|
38
33
|
__exportStar(require("./protocol-strategy/PostgresProtocolStrategyStreamed"), exports);
|
|
39
34
|
__exportStar(require("./protocol-strategy/ProtocolStrategy"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,0EAAwD;AACxD,0EAAwD;AACxD,yFAAuE;AACvE,gFAA8D;AAC9D,oFAAkE;AAClE,wEAAsD;AACtD,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}
|
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
import { IProtocolStrategy } from "./IProtocolStrategy";
|
|
2
|
-
import { Readable } from "node:stream";
|
|
3
2
|
import { DataSourceStream } from "../DataSourceStream";
|
|
4
3
|
import { HTTPRequestProtocolStrategy } from "./HTTPRequestProtocolStrategy";
|
|
5
|
-
|
|
6
|
-
export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy, IStreamingProtocolStrategy {
|
|
4
|
+
export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy {
|
|
7
5
|
private streamTransform;
|
|
8
6
|
/**
|
|
9
7
|
* @param streamTransform data will be piped to this stream if provided
|
|
10
8
|
*/
|
|
11
9
|
setStreamTransformer: (streamTransform: any) => HTTPRequestProtocolStrategyStreamed;
|
|
12
10
|
getData(): Promise<DataSourceStream>;
|
|
13
|
-
getBodyStream(): Promise<Readable>;
|
|
14
11
|
getRawData: () => Promise<any>;
|
|
15
12
|
}
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.HTTPRequestProtocolStrategyStreamed = void 0;
|
|
4
4
|
const errors_1 = require("@golemio/errors");
|
|
5
|
-
const node_zlib_1 = require("node:zlib");
|
|
6
5
|
const DataSourceStream_1 = require("../DataSourceStream");
|
|
7
6
|
const HTTPRequestProtocolStrategy_1 = require("./HTTPRequestProtocolStrategy");
|
|
8
7
|
class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.HTTPRequestProtocolStrategy {
|
|
@@ -17,8 +16,7 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
|
|
|
17
16
|
};
|
|
18
17
|
this.getRawData = async () => {
|
|
19
18
|
if (this.connectionSettings.compression) {
|
|
20
|
-
throw new errors_1.GeneralError("
|
|
21
|
-
"use DataSourceStreamedPipeline instead", this.constructor.name);
|
|
19
|
+
throw new errors_1.GeneralError("compressed resources are not supported in HTTPRequestProtocolStrategyStreamed yet", this.constructor.name);
|
|
22
20
|
}
|
|
23
21
|
try {
|
|
24
22
|
const result = await this.getResponseInternal();
|
|
@@ -99,33 +97,6 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
|
|
|
99
97
|
}
|
|
100
98
|
return outStream;
|
|
101
99
|
}
|
|
102
|
-
async getBodyStream() {
|
|
103
|
-
try {
|
|
104
|
-
const result = await this.getResponseInternal();
|
|
105
|
-
if (result.statusCode < 200 || result.statusCode >= 300) {
|
|
106
|
-
result.body?.resume?.(); // drain to avoid connection leak (some statuses have no body)
|
|
107
|
-
throw new errors_1.GeneralError(`Unexpected HTTP status ${result.statusCode}`, this.constructor.name);
|
|
108
|
-
}
|
|
109
|
-
let body = result.body;
|
|
110
|
-
if (this.connectionSettings.compression === "gzip") {
|
|
111
|
-
const encoding = String(result.headers?.["content-encoding"] ?? "").toLowerCase();
|
|
112
|
-
if (encoding !== "gzip") {
|
|
113
|
-
body?.resume?.();
|
|
114
|
-
throw new errors_1.GeneralError(`Expected gzip-encoded response but server returned Content-Encoding: "${encoding || "<none>"}"`, this.constructor.name);
|
|
115
|
-
}
|
|
116
|
-
body = body.pipe((0, node_zlib_1.createGunzip)());
|
|
117
|
-
}
|
|
118
|
-
return this.teeRawDataToStorage(body, {
|
|
119
|
-
statusCode: result.statusCode,
|
|
120
|
-
headers: result.headers,
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
catch (err) {
|
|
124
|
-
if (err instanceof errors_1.GeneralError)
|
|
125
|
-
throw err;
|
|
126
|
-
throw new errors_1.GeneralError("Error while getting body stream from server.", this.constructor.name, err);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
100
|
}
|
|
130
101
|
exports.HTTPRequestProtocolStrategyStreamed = HTTPRequestProtocolStrategyStreamed;
|
|
131
102
|
//# sourceMappingURL=HTTPRequestProtocolStrategyStreamed.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;
|
|
1
|
+
{"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;AACrE,0DAAuD;AACvD,+EAA4E;AAE5E,MAAa,mCAAoC,SAAQ,yDAA2B;IAApF;;QAGI;;WAEG;QACI,yBAAoB,GAAG,CAAC,eAAoB,EAAuC,EAAE;YACxF,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAsEK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,CAAC;gBACtC,MAAM,IAAI,qBAAY,CAClB,mFAAmF,EACnF,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;YACN,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAEhD,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,IAAI;iBACpB,CAAC;YACN,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAChG,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAtFU,KAAK,CAAC,OAAO;QAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,mCAAgB,CAAC;YACnC,UAAU,EAAE,IAAI;YAChB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC5C,IAAI,KAAK,YAAY,6BAAoB,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACJ,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,6BAA6B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3G,CAAC;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE,CAAC;oBACxC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC9C,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC9G,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACJ,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,aAAa,EAAE,CAAC;oBACjB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,uCAAuC;gBACvC,IAAI,CAAC,aAAa,EAAE,CAAC;oBACjB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACJ,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBACnG,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;CAoBJ;AAjGD,kFAiGC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { Readable } from "node:stream";
|
|
2
1
|
import { DataSourceStream } from "../DataSourceStream";
|
|
3
2
|
import { IProtocolStrategy, ProtocolStrategySettings } from "./IProtocolStrategy";
|
|
4
3
|
export declare class ProtocolStrategy implements IProtocolStrategy {
|
|
@@ -19,7 +18,6 @@ export declare class ProtocolStrategy implements IProtocolStrategy {
|
|
|
19
18
|
}>;
|
|
20
19
|
getLastModified: () => Promise<string | null>;
|
|
21
20
|
protected getRawData(...args: any): Promise<any | DataSourceStream>;
|
|
22
|
-
protected teeRawDataToStorage(bodyStream: Readable, meta?: unknown): Readable;
|
|
23
21
|
private prepareRawData;
|
|
24
22
|
private pushRawDataToS3Stream;
|
|
25
23
|
}
|
|
@@ -5,8 +5,7 @@ const CoreToken_1 = require("../../../helpers/ioc/CoreToken");
|
|
|
5
5
|
const ioc_1 = require("../../ioc");
|
|
6
6
|
const Di_1 = require("../../ioc/Di");
|
|
7
7
|
const errors_1 = require("@golemio/errors");
|
|
8
|
-
const
|
|
9
|
-
const promises_1 = require("node:stream/promises");
|
|
8
|
+
const stream_1 = require("stream");
|
|
10
9
|
const DataSourceStream_1 = require("../DataSourceStream");
|
|
11
10
|
class ProtocolStrategy {
|
|
12
11
|
constructor(settings) {
|
|
@@ -90,9 +89,9 @@ class ProtocolStrategy {
|
|
|
90
89
|
let dataSaveCalled = false;
|
|
91
90
|
let protocolDataStream;
|
|
92
91
|
// bit paranoid but ...
|
|
93
|
-
if (rawData && (rawData instanceof
|
|
92
|
+
if (rawData && (rawData instanceof stream_1.Readable || rawData.constructor.name === "Request") && rawData.on) {
|
|
94
93
|
protocolDataStream = rawData;
|
|
95
|
-
const S3Stream = new
|
|
94
|
+
const S3Stream = new stream_1.Readable({
|
|
96
95
|
objectMode: false,
|
|
97
96
|
read: () => {
|
|
98
97
|
return;
|
|
@@ -146,44 +145,6 @@ class ProtocolStrategy {
|
|
|
146
145
|
async getRawData(...args) {
|
|
147
146
|
throw new Error("getRawData: Method not implemented.");
|
|
148
147
|
}
|
|
149
|
-
teeRawDataToStorage(bodyStream, meta = {}) {
|
|
150
|
-
if (!this.storageEnabled || !this.saveRawDataWhitelist[this.caller]) {
|
|
151
|
-
return bodyStream;
|
|
152
|
-
}
|
|
153
|
-
const S3Stream = new node_stream_1.Readable({
|
|
154
|
-
objectMode: false,
|
|
155
|
-
read: () => {
|
|
156
|
-
return;
|
|
157
|
-
},
|
|
158
|
-
});
|
|
159
|
-
const consumer = new node_stream_1.PassThrough();
|
|
160
|
-
let consumerErrorFired = false;
|
|
161
|
-
(0, promises_1.pipeline)(bodyStream, consumer).catch((err) => {
|
|
162
|
-
if (!consumerErrorFired) {
|
|
163
|
-
this.logger.error(`Body stream source failed for ${this.caller}`, err);
|
|
164
|
-
}
|
|
165
|
-
});
|
|
166
|
-
let saveCalled = false;
|
|
167
|
-
consumer.on("data", async (chunk) => {
|
|
168
|
-
await this.pushRawDataToS3Stream(chunk, S3Stream);
|
|
169
|
-
if (!saveCalled) {
|
|
170
|
-
this.rawDataStorage.save(S3Stream, meta, this.caller);
|
|
171
|
-
saveCalled = true;
|
|
172
|
-
}
|
|
173
|
-
});
|
|
174
|
-
consumer.on("end", () => {
|
|
175
|
-
S3Stream.push(null);
|
|
176
|
-
});
|
|
177
|
-
consumer.on("error", (err) => {
|
|
178
|
-
consumerErrorFired = true;
|
|
179
|
-
this.logger.warn(`Raw data save to storage interrupted for ${this.caller}`, err);
|
|
180
|
-
S3Stream.push(null);
|
|
181
|
-
});
|
|
182
|
-
S3Stream.on("error", (err) => {
|
|
183
|
-
this.logger.error(`Error on saving raw data for ${this.caller}`, err);
|
|
184
|
-
});
|
|
185
|
-
return consumer;
|
|
186
|
-
}
|
|
187
148
|
}
|
|
188
149
|
exports.ProtocolStrategy = ProtocolStrategy;
|
|
189
150
|
//# sourceMappingURL=ProtocolStrategy.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;AACA,8DAAmD;AAInD,mCAAyC;AACzC,qCAAwD;AACxD,4CAAqE;AACrE,
|
|
1
|
+
{"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;AACA,8DAAmD;AAInD,mCAAyC;AACzC,qCAAwD;AACxD,4CAAqE;AACrE,mCAAkC;AAClC,0DAAuD;AAGvD,MAAa,gBAAgB;IASzB,YAAY,QAAa;QANjB,WAAM,GAAG,SAAS,CAAC;QAgBpB,kBAAa,GAAG,CAAC,SAAiB,oBAAoB,EAAE,EAAE;YAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,iBAAiB;QAC3C,CAAC,CAAC;QAMK,0BAAqB,GAAG,GAA6B,EAAE;YAC1D,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC,CAAC;QA8EK,oBAAe,GAAG,KAAK,IAA4B,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC,CAAC;QAMM,mBAAc,GAAG,KAAK,EAAE,IAAS,EAA0B,EAAE;YACjE,IAAI,OAAO,GAAkB,IAAI,CAAC;YAElC,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;gBACvB,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;gBACrC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;gBACvD,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBAAC,MAAM,CAAC;oBACL,IAAI,CAAC;gBACT,CAAC;YACL,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACxD,CAAC;gBAAC,MAAM,CAAC;oBACL,IAAI,CAAC;gBACT,CAAC;YACL,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACtC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,CAAC;YAED,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC;QAEM,0BAAqB,GAAG,KAAK,EAAE,IAAS,EAAE,QAAkB,EAAiB,EAAE;YACnF,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACzC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;wBAC9B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACxB,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,CAAC,MAAM;;qBAE3D,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QA5JE,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,MAAM,YAAY,GAAG,+BAA0B,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QAC/F,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC;QACvF,MAAM,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,MAAM,CAAC,CAAC;QACzF,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC,cAAc,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,cAAc,CAAC,CAAC;IAC5G,CAAC;IAMM,qBAAqB,CAAC,QAAa;QACtC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAMM,KAAK,CAAC,OAAO,CAAC,GAAG,IAAW;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,GAAG,IAAe;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,kBAAuB,CAAC;QAE5B,uBAAuB;QACvB,IAAI,OAAO,IAAI,CAAC,OAAO,YAAY,iBAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACnG,kBAAkB,GAAG,OAAO,CAAC;YAE7B,MAAM,QAAQ,GAAG,IAAI,iBAAQ,CAAC;gBAC1B,UAAU,EAAE,KAAK;gBACjB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC;gBACpC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,sFAAsF;YACtF,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAEjD,oGAAoG;gBACpG,IAAI,CAAC,cAAc,EAAE,CAAC;oBAClB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtD,cAAc,GAAG,IAAI,CAAC;gBAC1B,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC1C,IAAI,GAAG,YAAY,6BAAoB,EAAE,CAAC;oBACtC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;qBAAM,CAAC;oBACJ,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;gBACxG,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC9B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,OAAO;YACX,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAQ,EAAE,EAAE;gBAC3C,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;gBACjC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAChD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAChF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7C,CAAC;IACL,CAAC;IAMS,KAAK,CAAC,UAAU,CAAC,GAAG,IAAS;QACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;CAsDJ;AAvKD,4CAuKC"}
|
|
@@ -60,7 +60,6 @@ IntegrationEngineContainer.register(ContainerToken_1.ContainerToken.AmqpConnecto
|
|
|
60
60
|
poolMinConnections: Number(simpleConfig.getValue("env.POSTGRES_POOL_MIN_CONNECTIONS", 0)),
|
|
61
61
|
poolMaxConnections: Number(simpleConfig.getValue("env.POSTGRES_POOL_MAX_CONNECTIONS", 10)),
|
|
62
62
|
applicationName: "Integration Engine",
|
|
63
|
-
sslCaBase64: simpleConfig.getValue("env.POSTGRES_SSL_CA_BASE64", null),
|
|
64
63
|
}, c.resolve(CoreToken_1.CoreToken.Logger), {
|
|
65
64
|
define: {
|
|
66
65
|
freezeTableName: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../src/integration-engine/ioc/Di.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoF;AAEpF,4FAA+D;AAE/D,gFAAwE;AACxE,4FAAoF;AACpF,uFAA+E;AAC/E,mHAA2G;AAC3G,+HAAmH;AACnH,2DAAmD;AACnD,6CAAkD;AAElD,6FAA2F;AAC3F,mFAA4D;AAC5D,kFAAgF;AAChF,sDAAoD;AACpD,4EAA0E;AAC1E,uCAA6D;AAC7D,qDAAkD;AAElD,wBAAwB;AACxB,MAAM,gBAAgB,GAAG,EAAE,SAAS,EAAE,oBAAS,CAAC,SAAS,EAAE,CAAC;AAC5D,MAAM,0BAA0B,GAAG,oBAAe,CAAC,oBAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../src/integration-engine/ioc/Di.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoF;AAEpF,4FAA+D;AAE/D,gFAAwE;AACxE,4FAAoF;AACpF,uFAA+E;AAC/E,mHAA2G;AAC3G,+HAAmH;AACnH,2DAAmD;AACnD,6CAAkD;AAElD,6FAA2F;AAC3F,mFAA4D;AAC5D,kFAAgF;AAChF,sDAAoD;AACpD,4EAA0E;AAC1E,uCAA6D;AAC7D,qDAAkD;AAElD,wBAAwB;AACxB,MAAM,gBAAgB,GAAG,EAAE,SAAS,EAAE,oBAAS,CAAC,SAAS,EAAE,CAAC;AAC5D,MAAM,0BAA0B,GAAG,oBAAe,CAAC,oBAAoB,EAAE,CAAC;AAgIjE,gEAA0B;AA/HnC,YAAY;AAEZ,iBAAiB;AACjB,0BAA0B,CAAC,QAAQ,CAC/B,+BAAc,CAAC,MAAM,EACrB,EAAE,QAAQ,EAAE,+DAA8B,EAAE,EAC5C,gBAAgB,CACnB,CAAC;AAEF,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAiB,+BAAc,CAAC,MAAM,CAAC,CAAC;AACzF,0BAA0B,CAAC,gBAAgB,CACvC,qBAAS,CAAC,YAAY,EACtB,IAAI,sBAAY,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,uBAAY,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,CACrG,CAAC;AACF,0BAA0B,CAAC,QAAQ,CAAU,+BAAc,CAAC,MAAM,EAAE;IAChE,UAAU,EAAE,IAAA,iCAAsB,EAAC,GAAG,EAAE,CACpC,IAAA,uBAAY,EAAC;QACT,WAAW,EAAE,oBAAoB;QACjC,OAAO,EAAE,MAAM,CAAC,QAAQ;QACxB,QAAQ,EAAE,MAAM,CAAC,SAAS;KAC7B,CAAC,CACL;CACJ,CAAC,CAAC,gBAAgB,CACf,+BAAc,CAAC,aAAa,EAC5B,IAAA,8BAAmB,EAAC,MAAM,CAAC,QAAQ,EAAE,0BAA0B,CAAC,OAAO,CAAC,+BAAc,CAAC,MAAM,CAAC,CAAC,CAClG,CAAC;AACF,0BAA0B,CAAC,iBAAiB,CAAC,+BAAc,CAAC,qBAAqB,EAAE,6CAAqB,CAAC,CAAC,QAAQ,CAC9G,+BAAc,CAAC,aAAa,EAC5B;IACI,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAwB,+BAAc,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC;CAC5H,CACJ,CAAC;AACF,YAAY;AAEZ,qBAAqB;AACrB,0BAA0B,CAAC,QAAQ,CAAkB,+BAAc,CAAC,aAAa,EAAE;IAC/E,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QACtE,OAAO,IAAI,6BAAa,CACpB;YACI,sBAAsB,EAAE,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YAChE,kBAAkB,EAAE,YAAY,CAAC,QAAQ,CAAC,0BAA0B,CAAC;YACrE,6BAA6B,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,uCAAuC,EAAE,CAAC,CAAC,CAAC;SAC3G,EACD,CAAC,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CACvC,CAAC;IACN,CAAC,CAAC;CACL,CAAC;KACG,QAAQ,CAAC,qBAAS,CAAC,iBAAiB,EAAE;IACnC,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QACtE,OAAO,IAAI,qCAAiB,CACxB;YACI,gBAAgB,EAAE,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YAC5D,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACvF,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,mCAAmC,EAAE,CAAC,CAAC,CAAC;YACzF,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;YAC1F,eAAe,EAAE,oBAAoB;SACxC,EACD,CAAC,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,EACpC;YACI,MAAM,EAAE;gBACJ,eAAe,EAAE,IAAI;gBACrB,UAAU,EAAE,IAAI,EAAE,+BAA+B;gBACjD,SAAS,EAAE,YAAY;gBACvB,SAAS,EAAE,YAAY;gBACvB,WAAW,EAAE,IAAI,EAAE,wEAAwE;aAC9F;SACJ,CACJ,CAAC;IACN,CAAC,CAAC;CACL,CAAC;KACD,QAAQ,CAAC,+BAAc,CAAC,cAAc,EAAE;IACrC,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QACtE,OAAO,IAAI,mCAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACnH,CAAC,CAAC;CACL,CAAC;KACD,QAAQ,CAAC,+BAAc,CAAC,sBAAsB,EAAE,+CAAsB,EAAE,gBAAgB,CAAC;KACzF,QAAQ,CAAC,+BAAc,CAAC,cAAc,EAAE;IACrC,UAAU,EAAE,IAAA,iCAAsB,EAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAyB,+BAAc,CAAC,sBAAsB,CAAC,CAAC,OAAO,CAC1F;CACJ,CAAC;KACD,QAAQ,CAAC,+BAAc,CAAC,mBAAmB,EAAE;IAC1C,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,mBAAmB,GAAG,MAAM,CAAC,UAAU,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QAClF,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,qCAAqC,EAAE,IAAI,CAAC,CAAC;QACtF,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,aAAa,CAAC;QACpE,MAAM,cAAc,GAAG,IAAI,uDAA0B,CACjD;YACI,OAAO,EAAE,mBAAmB;YAC5B,QAAQ,EAAE,mBAAmB;gBACzB,CAAC,CAAC;oBACI,CAAC,+CAAoB,CAAC,KAAK,CAAC,EAAE;wBAC1B,QAAQ,EACJ,MAAM,CAAC,QAAQ,CAAC,qBAAqB,EAAE,IAAI,CAAC;4BAC5C,MAAM,CAAC,QAAQ,CAAC,2BAA2B,EAAE,IAAI,CAAC;wBACtD,QAAQ,EACJ,MAAM,CAAC,QAAQ,CAAC,qBAAqB,EAAE,IAAI,CAAC;4BAC5C,MAAM,CAAC,QAAQ,CAAC,2BAA2B,EAAE,IAAI,CAAC;wBACtD,YAAY,EACR,MAAM,CAAC,QAAQ,CAAC,yBAAyB,EAAE,IAAI,CAAC;4BAChD,MAAM,CAAC,QAAQ,CAAC,+BAA+B,EAAE,IAAI,CAAC;wBAC1D,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC;wBACnD,eAAe,EAAE,MAAM,CAAC,QAAQ,CAC5B,MAAM,CAAC,QAAQ,CAAC,2CAA2C,EAAE,KAAK,CAAC,CACtE;wBACD,uBAAuB,EAAE,MAAM,CAAC,QAAQ,CAAC,gCAAgC,EAAE,KAAK,CAAC;wBACjF,qCAAqC;wBACrC,gBAAgB,EAAE,gBAAgB;wBAClC,mBAAmB,EAAE,SAAS,IAAI,CAAC,CAAC,gBAAgB;qBACvD;iBACJ;gBACH,CAAC,CAAC,EAAE;SACX,EACD,CAAC,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CACvC,CAAC;QAEF,OAAO,cAAc,CAAC,UAAU,CAAC,+CAAoB,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC,CAAC;CACL,CAAC;KACD,QAAQ,CAAa,+BAAc,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,uBAAU,EAAE,EAAE,gBAAgB,CAAC;KAC3F,iBAAiB,CAAC,+BAAc,CAAC,cAAc,EAAE,wBAAc,CAAC,CAAC"}
|
|
@@ -48,7 +48,6 @@ OutputGatewayContainer.register(CoreToken_1.CoreToken.PostgresConnector, {
|
|
|
48
48
|
poolMinConnections: Number(simpleConfig.getValue("env.POSTGRES_POOL_MIN_CONNECTIONS", 0)),
|
|
49
49
|
poolMaxConnections: Number(simpleConfig.getValue("env.POSTGRES_POOL_MAX_CONNECTIONS", 5)),
|
|
50
50
|
applicationName: "Output Gateway",
|
|
51
|
-
sslCaBase64: simpleConfig.getValue("env.POSTGRES_SSL_CA_BASE64", null),
|
|
52
51
|
}, c.resolve(CoreToken_1.CoreToken.Logger), {
|
|
53
52
|
define: {
|
|
54
53
|
freezeTableName: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../src/output-gateway/ioc/Di.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoF;AAEpF,4FAA+D;AAC/D,4FAAoF;AAEpF,uFAA+E;AAC/E,2DAAmD;AACnD,6CAAkD;AAClD,oEAAkE;AAClE,sFAAoF;AACpF,qFAAmF;AACnF,uCAAkD;AAClD,qDAAkD;AAElD,wBAAwB;AACxB,MAAM,sBAAsB,GAAG,oBAAe,CAAC,oBAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"Di.js","sourceRoot":"","sources":["../../../src/output-gateway/ioc/Di.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoF;AAEpF,4FAA+D;AAC/D,4FAAoF;AAEpF,uFAA+E;AAC/E,2DAAmD;AACnD,6CAAkD;AAClD,oEAAkE;AAClE,sFAAoF;AACpF,qFAAmF;AACnF,uCAAkD;AAClD,qDAAkD;AAElD,wBAAwB;AACxB,MAAM,sBAAsB,GAAG,oBAAe,CAAC,oBAAoB,EAAE,CAAC;AAmE7D,wDAAsB;AAlE/B,YAAY;AAEZ,iBAAiB;AACjB,sBAAsB,CAAC,iBAAiB,CAAC,+BAAc,CAAC,MAAM,EAAE,uDAA0B,CAAC,CAAC;AAC5F,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAA6B,+BAAc,CAAC,MAAM,CAAC,CAAC;AACjG,MAAM,YAAY,GAAG,IAAI,sBAAY,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAEzE,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,MAAM,EAAE,CAAC;IAC9C,MAAM,YAAY,GAAG,IAAI,uBAAY,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC;IAE9D,YAAY,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,sBAAsB,CAAC,gBAAgB,CAAgB,qBAAS,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;AAC7F,sBAAsB,CAAC,QAAQ,CAAU,+BAAc,CAAC,MAAM,EAAE;IAC5D,UAAU,EAAE,IAAA,iCAAsB,EAAC,GAAG,EAAE,CACpC,IAAA,uBAAY,EAAC;QACT,WAAW,EAAE,gBAAgB;QAC7B,OAAO,EAAE,MAAM,CAAC,QAAQ;QACxB,QAAQ,EAAE,MAAM,CAAC,SAAS;KAC7B,CAAC,CACL;CACJ,CAAC,CAAC,gBAAgB,CACf,+BAAc,CAAC,aAAa,EAC5B,IAAA,8BAAmB,EAAC,MAAM,CAAC,QAAQ,EAAE,sBAAsB,CAAC,OAAO,CAAU,+BAAc,CAAC,MAAM,CAAC,CAAC,CACvG,CAAC;AACF,sBAAsB,CAAC,iBAAiB,CAAC,+BAAc,CAAC,qBAAqB,EAAE,6CAAqB,CAAC,CAAC;AACtG,sBAAsB,CAAC,iBAAiB,CAAC,+BAAc,CAAC,8BAA8B,EAAE,+DAA8B,CAAC,CAAC;AACxH,YAAY;AAEZ,qBAAqB;AACrB,sBAAsB,CAAC,QAAQ,CAAC,qBAAS,CAAC,iBAAiB,EAAE;IACzD,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QACtE,OAAO,IAAI,qCAAiB,CACxB;YACI,gBAAgB,EAAE,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YAC5D,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACvF,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,mCAAmC,EAAE,CAAC,CAAC,CAAC;YACzF,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,mCAAmC,EAAE,CAAC,CAAC,CAAC;YACzF,eAAe,EAAE,gBAAgB;SACpC,EACD,CAAC,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,EACpC;YACI,MAAM,EAAE;gBACJ,eAAe,EAAE,IAAI;gBACrB,UAAU,EAAE,KAAK;aACpB;SACJ,CACJ,CAAC;IACN,CAAC,CAAC;CACL,CAAC,CAAC,QAAQ,CAAC,+BAAc,CAAC,mBAAmB,EAAE;IAC5C,UAAU,EAAE,IAAA,iCAAsB,EAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,OAAO,CAAqB,qBAAS,CAAC,iBAAiB,CAAC,CAAC,OAAO,EAAE,CAAC;CAC9H,CAAC,CAAC;AAEH,sBAAsB,CAAC,QAAQ,CAAC,+BAAc,CAAC,cAAc,EAAE;IAC3D,UAAU,EAAE,IAAA,iCAAsB,EAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QACtE,IAAI,YAAY,CAAC,UAAU,CAAC,kBAAkB,EAAE,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,IAAI,mCAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QACnH,CAAC;IACL,CAAC,CAAC;CACL,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/core",
|
|
3
|
-
"version": "4.0.0-
|
|
3
|
+
"version": "4.0.0-rc.2674851974",
|
|
4
4
|
"description": "Golemio Core Module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -38,7 +38,6 @@
|
|
|
38
38
|
"@types/chai-as-promised": "^8.0.2",
|
|
39
39
|
"@types/content-type": "^1.1.3",
|
|
40
40
|
"@types/debug": "0.0.31",
|
|
41
|
-
"@types/jsonstream": "^0.8.33",
|
|
42
41
|
"@types/mocha": "^10.0.10",
|
|
43
42
|
"@types/node": "^24.12.4",
|
|
44
43
|
"@types/sinon": "^21.0.0",
|
|
@@ -78,7 +77,7 @@
|
|
|
78
77
|
"@google-cloud/storage": "^7.19.0",
|
|
79
78
|
"@opensearch-project/opensearch": "^3.6.0",
|
|
80
79
|
"@opentelemetry/api": "^1.9.0",
|
|
81
|
-
"@opentelemetry/exporter-trace-otlp-grpc": "^0.
|
|
80
|
+
"@opentelemetry/exporter-trace-otlp-grpc": "^0.204.0",
|
|
82
81
|
"@opentelemetry/instrumentation-amqplib": "^0.50.0",
|
|
83
82
|
"@opentelemetry/instrumentation-aws-sdk": "^0.56.0",
|
|
84
83
|
"@opentelemetry/instrumentation-express": "^0.52.0",
|
|
@@ -92,7 +91,7 @@
|
|
|
92
91
|
"@types/geojson": "^7946.0.10",
|
|
93
92
|
"@types/luxon": "^2.3.1",
|
|
94
93
|
"@types/qs": "^6.9.18",
|
|
95
|
-
"ajv": "^8.
|
|
94
|
+
"ajv": "^8.10.0",
|
|
96
95
|
"amqplib": "^0.10.8",
|
|
97
96
|
"basic-ftp": "^6.0.1",
|
|
98
97
|
"class-transformer": "^0.5.1",
|
|
@@ -107,7 +106,6 @@
|
|
|
107
106
|
"html-entities": "^2.4.0",
|
|
108
107
|
"iconv-lite": "^0.6.2",
|
|
109
108
|
"ioredis": "^5.4.1",
|
|
110
|
-
"JSONStream": "^1.3.5",
|
|
111
109
|
"lightship": "9.0.2",
|
|
112
110
|
"luxon": "^2.5.2",
|
|
113
111
|
"moment-timezone": "^0.5.26",
|
|
@@ -125,7 +123,7 @@
|
|
|
125
123
|
"sequelize": "^6.37.8",
|
|
126
124
|
"ssh2-sftp-client": "^7.1.0",
|
|
127
125
|
"tsyringe": "^4.8.0",
|
|
128
|
-
"undici": "^7.
|
|
126
|
+
"undici": "^7.24.4",
|
|
129
127
|
"xml2js": "^0.5.0",
|
|
130
128
|
"yauzl": "^2.10.0"
|
|
131
129
|
},
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { Writable } from "node:stream";
|
|
2
|
-
export interface BatchWritableOptions<T> {
|
|
3
|
-
batchSize: number;
|
|
4
|
-
onFlush: (batch: T[]) => Promise<void>;
|
|
5
|
-
}
|
|
6
|
-
export declare class BatchWritable<T> extends Writable {
|
|
7
|
-
private batch;
|
|
8
|
-
private readonly batchSize;
|
|
9
|
-
private readonly onFlush;
|
|
10
|
-
constructor(options: BatchWritableOptions<T>);
|
|
11
|
-
_write(item: T, _encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
|
|
12
|
-
_final(callback: (err?: Error | null) => void): void;
|
|
13
|
-
private flush;
|
|
14
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BatchWritable = void 0;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
|
-
class BatchWritable extends node_stream_1.Writable {
|
|
6
|
-
constructor(options) {
|
|
7
|
-
super({
|
|
8
|
-
objectMode: true,
|
|
9
|
-
});
|
|
10
|
-
this.batch = [];
|
|
11
|
-
this.batchSize = options.batchSize;
|
|
12
|
-
this.onFlush = options.onFlush;
|
|
13
|
-
}
|
|
14
|
-
_write(item, _encoding, callback) {
|
|
15
|
-
this.batch.push(item);
|
|
16
|
-
if (this.batch.length < this.batchSize) {
|
|
17
|
-
callback();
|
|
18
|
-
return;
|
|
19
|
-
}
|
|
20
|
-
this.flush(callback);
|
|
21
|
-
}
|
|
22
|
-
_final(callback) {
|
|
23
|
-
if (this.batch.length === 0) {
|
|
24
|
-
callback();
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
this.flush(callback);
|
|
28
|
-
}
|
|
29
|
-
async flush(callback) {
|
|
30
|
-
const fullBatch = this.batch;
|
|
31
|
-
this.batch = [];
|
|
32
|
-
try {
|
|
33
|
-
await this.onFlush(fullBatch);
|
|
34
|
-
callback();
|
|
35
|
-
}
|
|
36
|
-
catch (err) {
|
|
37
|
-
callback(err instanceof Error ? err : new Error(String(err)));
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
exports.BatchWritable = BatchWritable;
|
|
42
|
-
//# sourceMappingURL=BatchWritable.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"BatchWritable.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/BatchWritable.ts"],"names":[],"mappings":";;;AAAA,6CAAuC;AAOvC,MAAa,aAAiB,SAAQ,sBAAQ;IAK1C,YAAY,OAAgC;QACxC,KAAK,CAAC;YACF,UAAU,EAAE,IAAI;SACnB,CAAC,CAAC;QAPC,UAAK,GAAQ,EAAE,CAAC;QAQpB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAEe,MAAM,CAAC,IAAO,EAAE,SAAyB,EAAE,QAAsC;QAC7F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,QAAQ,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEe,MAAM,CAAC,QAAsC;QACzD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,QAAsC;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;CACJ;AAxCD,sCAwCC"}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { IDataSource } from "./IDataSource";
|
|
2
|
-
import { IValidator } from "@golemio/validator";
|
|
3
|
-
import { Transform } from "node:stream";
|
|
4
|
-
import { DataSource } from "./DataSource";
|
|
5
|
-
import { IStreamingProtocolStrategy } from "./protocol-strategy/IStreamingProtocolStrategy";
|
|
6
|
-
export interface DataSourceStreamedPipelineOptions {
|
|
7
|
-
validator?: IValidator | null;
|
|
8
|
-
batchSize?: number;
|
|
9
|
-
timeoutMs?: number;
|
|
10
|
-
}
|
|
11
|
-
export declare class DataSourceStreamedPipeline<T extends object = any> extends DataSource implements IDataSource {
|
|
12
|
-
static readonly DEFAULT_BATCH_SIZE: number;
|
|
13
|
-
static readonly DEFAULT_PIPELINE_TIMEOUT_MS: number;
|
|
14
|
-
private readonly streamTransformer;
|
|
15
|
-
private readonly batchSize;
|
|
16
|
-
private readonly timeoutMs;
|
|
17
|
-
private dataProcessor;
|
|
18
|
-
constructor(name: string, streamingStrategy: IStreamingProtocolStrategy, streamTransformer: Transform, options?: DataSourceStreamedPipelineOptions);
|
|
19
|
-
private get streamingStrategy();
|
|
20
|
-
setDataProcessor(fn: (batch: T[]) => Promise<void>): this;
|
|
21
|
-
proceed(): Promise<void>;
|
|
22
|
-
private createBatchWriter;
|
|
23
|
-
private validateAndProcess;
|
|
24
|
-
}
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DataSourceStreamedPipeline = void 0;
|
|
4
|
-
const logger_1 = require("../../helpers/logger");
|
|
5
|
-
const errors_1 = require("@golemio/errors");
|
|
6
|
-
const promises_1 = require("node:stream/promises");
|
|
7
|
-
const BatchWritable_1 = require("./BatchWritable");
|
|
8
|
-
const DataSource_1 = require("./DataSource");
|
|
9
|
-
class DataSourceStreamedPipeline extends DataSource_1.DataSource {
|
|
10
|
-
constructor(name, streamingStrategy, streamTransformer, options = {}) {
|
|
11
|
-
super(name, streamingStrategy, null, options.validator ?? null);
|
|
12
|
-
this.dataProcessor = null;
|
|
13
|
-
this.streamTransformer = streamTransformer;
|
|
14
|
-
this.batchSize = options.batchSize ?? DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE;
|
|
15
|
-
this.timeoutMs = options.timeoutMs ?? DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS;
|
|
16
|
-
}
|
|
17
|
-
get streamingStrategy() {
|
|
18
|
-
return this.protocolStrategy;
|
|
19
|
-
}
|
|
20
|
-
setDataProcessor(fn) {
|
|
21
|
-
this.dataProcessor = fn;
|
|
22
|
-
return this;
|
|
23
|
-
}
|
|
24
|
-
async proceed() {
|
|
25
|
-
if (!this.dataProcessor) {
|
|
26
|
-
throw new errors_1.GeneralError("No data processor registered. Call setDataProcessor() before proceed().", this.name);
|
|
27
|
-
}
|
|
28
|
-
const bodyStream = await this.streamingStrategy.getBodyStream();
|
|
29
|
-
let totalRecords = 0;
|
|
30
|
-
const batchWriter = this.createBatchWriter((processedCount) => {
|
|
31
|
-
totalRecords += processedCount;
|
|
32
|
-
});
|
|
33
|
-
try {
|
|
34
|
-
await (0, promises_1.pipeline)(bodyStream, this.streamTransformer, batchWriter, {
|
|
35
|
-
signal: AbortSignal.timeout(this.timeoutMs),
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
catch (err) {
|
|
39
|
-
bodyStream.destroy();
|
|
40
|
-
if (err instanceof errors_1.GeneralError)
|
|
41
|
-
throw err;
|
|
42
|
-
throw new errors_1.GeneralError("DataSourceStreamedPipeline pipeline failed.", this.name, err);
|
|
43
|
-
}
|
|
44
|
-
this.loggerEmitter.emit(logger_1.LoggerEventType.NumberOfRecords, { name: this.name, numberOfRecords: totalRecords });
|
|
45
|
-
}
|
|
46
|
-
createBatchWriter(onBatchProcessed) {
|
|
47
|
-
const batchSize = this.batchSize;
|
|
48
|
-
return new BatchWritable_1.BatchWritable({
|
|
49
|
-
batchSize,
|
|
50
|
-
onFlush: async (batch) => {
|
|
51
|
-
const processedCount = batch.length;
|
|
52
|
-
await this.validateAndProcess(batch);
|
|
53
|
-
onBatchProcessed(processedCount);
|
|
54
|
-
},
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
async validateAndProcess(batch) {
|
|
58
|
-
if (this.validator) {
|
|
59
|
-
try {
|
|
60
|
-
await this.validator.Validate(batch);
|
|
61
|
-
}
|
|
62
|
-
catch (err) {
|
|
63
|
-
throw new errors_1.GeneralError("Error while validating source data.", this.name, err);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
await this.dataProcessor(batch);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
exports.DataSourceStreamedPipeline = DataSourceStreamedPipeline;
|
|
70
|
-
DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE = process.env.DATA_BATCH_SIZE ? Number(process.env.DATA_BATCH_SIZE) : 1000;
|
|
71
|
-
DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
72
|
-
//# sourceMappingURL=DataSourceStreamedPipeline.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"DataSourceStreamedPipeline.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamedPipeline.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAElD,4CAA+C;AAG/C,mDAAgD;AAChD,mDAAgD;AAChD,6CAA0C;AAU1C,MAAa,0BAAmD,SAAQ,uBAAU;IAU9E,YACI,IAAY,EACZ,iBAA6C,EAC7C,iBAA4B,EAC5B,UAA6C,EAAE;QAE/C,KAAK,CAAC,IAAI,EAAE,iBAAiD,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAR5F,kBAAa,GAA2C,IAAI,CAAC;QASjE,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,kBAAkB,CAAC;QACpF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,2BAA2B,CAAC;IACjG,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,gBAAyD,CAAC;IAC1E,CAAC;IAEM,gBAAgB,CAAC,EAAiC;QACrD,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAY,CAAC,yEAAyE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;QAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,cAAc,EAAE,EAAE;YAC1D,YAAY,IAAI,cAAc,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC5D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC9C,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,6CAA6C,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC,CAAC;IACjH,CAAC;IAEO,iBAAiB,CAAC,gBAAyC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,OAAO,IAAI,6BAAa,CAAI;YACxB,SAAS;YACT,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACrB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;gBACpC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBACrC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACrC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAU;QACvC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAClF,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;;AA7EL,gEA8EC;AA7E0B,6CAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,AAA3E,CAA4E;AAC9F,sDAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,AAAjB,CAAkB"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.JSONStreamTransformer = void 0;
|
|
4
|
-
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
5
|
-
const JSONStream = require("JSONStream");
|
|
6
|
-
class JSONStreamTransformer {
|
|
7
|
-
static parse(path) {
|
|
8
|
-
return JSONStream.parse(path);
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
exports.JSONStreamTransformer = JSONStreamTransformer;
|
|
12
|
-
//# sourceMappingURL=JSONStreamTransformer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"JSONStreamTransformer.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.ts"],"names":[],"mappings":";;;AAGA,8DAA8D;AAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAA4B,CAAC;AACpE,MAAa,qBAAqB;IAC9B,MAAM,CAAC,KAAK,CAAC,IAAY;QACrB,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;IAC1D,CAAC;CACJ;AAJD,sDAIC"}
|
package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"IStreamingProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.ts"],"names":[],"mappings":""}
|