@golemio/core 4.0.1 → 4.1.0-rc.2893266968
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/integration-engine/datasources/BatchWritable.d.ts +14 -0
- package/dist/integration-engine/datasources/BatchWritable.js +42 -0
- package/dist/integration-engine/datasources/BatchWritable.js.map +1 -0
- package/dist/integration-engine/datasources/DataSourceStreamedPipeline.d.ts +24 -0
- package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js +72 -0
- package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js.map +1 -0
- package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.d.ts +4 -0
- package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js +12 -0
- package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js.map +1 -0
- package/dist/integration-engine/datasources/index.d.ts +5 -0
- package/dist/integration-engine/datasources/index.js +5 -0
- package/dist/integration-engine/datasources/index.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.d.ts +4 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js +30 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.d.ts +4 -0
- package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js +3 -0
- package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map +1 -0
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.d.ts +2 -0
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +42 -3
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
- package/package.json +7 -4
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Writable } from "node:stream";
|
|
2
|
+
export interface BatchWritableOptions<T> {
|
|
3
|
+
batchSize: number;
|
|
4
|
+
onFlush: (batch: T[]) => Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
export declare class BatchWritable<T> extends Writable {
|
|
7
|
+
private batch;
|
|
8
|
+
private readonly batchSize;
|
|
9
|
+
private readonly onFlush;
|
|
10
|
+
constructor(options: BatchWritableOptions<T>);
|
|
11
|
+
_write(item: T, _encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
|
|
12
|
+
_final(callback: (err?: Error | null) => void): void;
|
|
13
|
+
private flush;
|
|
14
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BatchWritable = void 0;
|
|
4
|
+
const node_stream_1 = require("node:stream");
|
|
5
|
+
class BatchWritable extends node_stream_1.Writable {
|
|
6
|
+
constructor(options) {
|
|
7
|
+
super({
|
|
8
|
+
objectMode: true,
|
|
9
|
+
});
|
|
10
|
+
this.batch = [];
|
|
11
|
+
this.batchSize = options.batchSize;
|
|
12
|
+
this.onFlush = options.onFlush;
|
|
13
|
+
}
|
|
14
|
+
_write(item, _encoding, callback) {
|
|
15
|
+
this.batch.push(item);
|
|
16
|
+
if (this.batch.length < this.batchSize) {
|
|
17
|
+
callback();
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
this.flush(callback);
|
|
21
|
+
}
|
|
22
|
+
_final(callback) {
|
|
23
|
+
if (this.batch.length === 0) {
|
|
24
|
+
callback();
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
this.flush(callback);
|
|
28
|
+
}
|
|
29
|
+
async flush(callback) {
|
|
30
|
+
const fullBatch = this.batch;
|
|
31
|
+
this.batch = [];
|
|
32
|
+
try {
|
|
33
|
+
await this.onFlush(fullBatch);
|
|
34
|
+
callback();
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
callback(err instanceof Error ? err : new Error(String(err)));
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
exports.BatchWritable = BatchWritable;
|
|
42
|
+
//# sourceMappingURL=BatchWritable.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BatchWritable.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/BatchWritable.ts"],"names":[],"mappings":";;;AAAA,6CAAuC;AAOvC,MAAa,aAAiB,SAAQ,sBAAQ;IAK1C,YAAY,OAAgC;QACxC,KAAK,CAAC;YACF,UAAU,EAAE,IAAI;SACnB,CAAC,CAAC;QAPC,UAAK,GAAQ,EAAE,CAAC;QAQpB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAEe,MAAM,CAAC,IAAO,EAAE,SAAyB,EAAE,QAAsC;QAC7F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,QAAQ,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEe,MAAM,CAAC,QAAsC;QACzD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,QAAsC;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;CACJ;AAxCD,sCAwCC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { IDataSource } from "./IDataSource";
|
|
2
|
+
import { IValidator } from "@golemio/validator";
|
|
3
|
+
import { Transform } from "node:stream";
|
|
4
|
+
import { DataSource } from "./DataSource";
|
|
5
|
+
import { IStreamingProtocolStrategy } from "./protocol-strategy/IStreamingProtocolStrategy";
|
|
6
|
+
export interface DataSourceStreamedPipelineOptions {
|
|
7
|
+
validator?: IValidator | null;
|
|
8
|
+
batchSize?: number;
|
|
9
|
+
timeoutMs?: number;
|
|
10
|
+
}
|
|
11
|
+
export declare class DataSourceStreamedPipeline<T extends object = any> extends DataSource implements IDataSource {
|
|
12
|
+
static readonly DEFAULT_BATCH_SIZE: number;
|
|
13
|
+
static readonly DEFAULT_PIPELINE_TIMEOUT_MS: number;
|
|
14
|
+
private readonly streamTransformer;
|
|
15
|
+
private readonly batchSize;
|
|
16
|
+
private readonly timeoutMs;
|
|
17
|
+
private dataProcessor;
|
|
18
|
+
constructor(name: string, streamingStrategy: IStreamingProtocolStrategy, streamTransformer: Transform, options?: DataSourceStreamedPipelineOptions);
|
|
19
|
+
private get streamingStrategy();
|
|
20
|
+
setDataProcessor(fn: (batch: T[]) => Promise<void>): this;
|
|
21
|
+
proceed(): Promise<void>;
|
|
22
|
+
private createBatchWriter;
|
|
23
|
+
private validateAndProcess;
|
|
24
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DataSourceStreamedPipeline = void 0;
|
|
4
|
+
const logger_1 = require("../../helpers/logger");
|
|
5
|
+
const errors_1 = require("@golemio/errors");
|
|
6
|
+
const promises_1 = require("node:stream/promises");
|
|
7
|
+
const BatchWritable_1 = require("./BatchWritable");
|
|
8
|
+
const DataSource_1 = require("./DataSource");
|
|
9
|
+
class DataSourceStreamedPipeline extends DataSource_1.DataSource {
|
|
10
|
+
constructor(name, streamingStrategy, streamTransformer, options = {}) {
|
|
11
|
+
super(name, streamingStrategy, null, options.validator ?? null);
|
|
12
|
+
this.dataProcessor = null;
|
|
13
|
+
this.streamTransformer = streamTransformer;
|
|
14
|
+
this.batchSize = options.batchSize ?? DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE;
|
|
15
|
+
this.timeoutMs = options.timeoutMs ?? DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS;
|
|
16
|
+
}
|
|
17
|
+
get streamingStrategy() {
|
|
18
|
+
return this.protocolStrategy;
|
|
19
|
+
}
|
|
20
|
+
setDataProcessor(fn) {
|
|
21
|
+
this.dataProcessor = fn;
|
|
22
|
+
return this;
|
|
23
|
+
}
|
|
24
|
+
async proceed() {
|
|
25
|
+
if (!this.dataProcessor) {
|
|
26
|
+
throw new errors_1.GeneralError("No data processor registered. Call setDataProcessor() before proceed().", this.name);
|
|
27
|
+
}
|
|
28
|
+
const bodyStream = await this.streamingStrategy.getBodyStream();
|
|
29
|
+
let totalRecords = 0;
|
|
30
|
+
const batchWriter = this.createBatchWriter((processedCount) => {
|
|
31
|
+
totalRecords += processedCount;
|
|
32
|
+
});
|
|
33
|
+
try {
|
|
34
|
+
await (0, promises_1.pipeline)(bodyStream, this.streamTransformer, batchWriter, {
|
|
35
|
+
signal: AbortSignal.timeout(this.timeoutMs),
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
bodyStream.destroy();
|
|
40
|
+
if (err instanceof errors_1.GeneralError)
|
|
41
|
+
throw err;
|
|
42
|
+
throw new errors_1.GeneralError("DataSourceStreamedPipeline pipeline failed.", this.name, err);
|
|
43
|
+
}
|
|
44
|
+
this.loggerEmitter.emit(logger_1.LoggerEventType.NumberOfRecords, { name: this.name, numberOfRecords: totalRecords });
|
|
45
|
+
}
|
|
46
|
+
createBatchWriter(onBatchProcessed) {
|
|
47
|
+
const batchSize = this.batchSize;
|
|
48
|
+
return new BatchWritable_1.BatchWritable({
|
|
49
|
+
batchSize,
|
|
50
|
+
onFlush: async (batch) => {
|
|
51
|
+
const processedCount = batch.length;
|
|
52
|
+
await this.validateAndProcess(batch);
|
|
53
|
+
onBatchProcessed(processedCount);
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async validateAndProcess(batch) {
|
|
58
|
+
if (this.validator) {
|
|
59
|
+
try {
|
|
60
|
+
await this.validator.Validate(batch);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
throw new errors_1.GeneralError("Error while validating source data.", this.name, err);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
await this.dataProcessor(batch);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
exports.DataSourceStreamedPipeline = DataSourceStreamedPipeline;
|
|
70
|
+
DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE = process.env.DATA_BATCH_SIZE ? Number(process.env.DATA_BATCH_SIZE) : 1000;
|
|
71
|
+
DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
72
|
+
//# sourceMappingURL=DataSourceStreamedPipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DataSourceStreamedPipeline.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamedPipeline.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAElD,4CAA+C;AAG/C,mDAAgD;AAChD,mDAAgD;AAChD,6CAA0C;AAU1C,MAAa,0BAAmD,SAAQ,uBAAU;IAU9E,YACI,IAAY,EACZ,iBAA6C,EAC7C,iBAA4B,EAC5B,UAA6C,EAAE;QAE/C,KAAK,CAAC,IAAI,EAAE,iBAAiD,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAR5F,kBAAa,GAA2C,IAAI,CAAC;QASjE,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,kBAAkB,CAAC;QACpF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,2BAA2B,CAAC;IACjG,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,gBAAyD,CAAC;IAC1E,CAAC;IAEM,gBAAgB,CAAC,EAAiC;QACrD,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAY,CAAC,yEAAyE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;QAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,cAAc,EAAE,EAAE;YAC1D,YAAY,IAAI,cAAc,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC5D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC9C,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,6CAA6C,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC,CAAC;IACjH,CAAC;IAEO,iBAAiB,CAAC,gBAAyC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,OAAO,IAAI,6BAAa,CAAI;YACxB,SAAS;YACT,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACrB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;gBACpC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBACrC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACrC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAU;QACvC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAClF,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;;AA7EL,gEA8EC;AA7E0B,6CAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,AAA3E,CAA4E;AAC9F,sDAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,AAAjB,CAAkB"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JSONStreamTransformer = void 0;
|
|
4
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
5
|
+
const JSONStream = require("JSONStream");
|
|
6
|
+
class JSONStreamTransformer {
|
|
7
|
+
static parse(path) {
|
|
8
|
+
return JSONStream.parse(path);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
exports.JSONStreamTransformer = JSONStreamTransformer;
|
|
12
|
+
//# sourceMappingURL=JSONStreamTransformer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"JSONStreamTransformer.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.ts"],"names":[],"mappings":";;;AAGA,8DAA8D;AAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAA4B,CAAC;AACpE,MAAa,qBAAqB;IAC9B,MAAM,CAAC,KAAK,CAAC,IAAY;QACrB,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;IAC1D,CAAC;CACJ;AAJD,sDAIC"}
|
|
@@ -1,17 +1,22 @@
|
|
|
1
|
+
export * from "./BatchWritable";
|
|
1
2
|
export * from "./DataSource";
|
|
2
3
|
export * from "./DataSourceStream";
|
|
3
4
|
export * from "./DataSourceStreamManager";
|
|
4
5
|
export * from "./DataSourceStreamed";
|
|
6
|
+
export * from "./DataSourceStreamedPipeline";
|
|
5
7
|
export * from "./IDataSource";
|
|
6
8
|
export * from "./datatype-strategy/CSVDataTypeStrategy";
|
|
7
9
|
export * from "./datatype-strategy/IDataTypeStrategy";
|
|
8
10
|
export * from "./datatype-strategy/JSONDataTypeStrategy";
|
|
11
|
+
export * from "./datatype-strategy/JSONStreamTransformer";
|
|
9
12
|
export * from "./datatype-strategy/XMLDataTypeStrategy";
|
|
10
13
|
export * from "./protocol-strategy/FTPProtocolStrategy";
|
|
14
|
+
export * from "./protocol-strategy/HTTPRequestProtocolStrategyStreamed";
|
|
11
15
|
export * from "./protocol-strategy/GoogleCloudStorageProtocolStrategy";
|
|
12
16
|
export * from "./protocol-strategy/HTTPFetchProtocolStrategy";
|
|
13
17
|
export * from "./protocol-strategy/PaginatedHTTPProtocolStrategy";
|
|
14
18
|
export * from "./protocol-strategy/IProtocolStrategy";
|
|
19
|
+
export * from "./protocol-strategy/IStreamingProtocolStrategy";
|
|
15
20
|
export * from "./protocol-strategy/PostgresProtocolStrategy";
|
|
16
21
|
export * from "./protocol-strategy/PostgresProtocolStrategyStreamed";
|
|
17
22
|
export * from "./protocol-strategy/ProtocolStrategy";
|
|
@@ -15,20 +15,25 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
/* ie/datasources/index.ts */
|
|
18
|
+
__exportStar(require("./BatchWritable"), exports);
|
|
18
19
|
__exportStar(require("./DataSource"), exports);
|
|
19
20
|
__exportStar(require("./DataSourceStream"), exports);
|
|
20
21
|
__exportStar(require("./DataSourceStreamManager"), exports);
|
|
21
22
|
__exportStar(require("./DataSourceStreamed"), exports);
|
|
23
|
+
__exportStar(require("./DataSourceStreamedPipeline"), exports);
|
|
22
24
|
__exportStar(require("./IDataSource"), exports);
|
|
23
25
|
__exportStar(require("./datatype-strategy/CSVDataTypeStrategy"), exports);
|
|
24
26
|
__exportStar(require("./datatype-strategy/IDataTypeStrategy"), exports);
|
|
25
27
|
__exportStar(require("./datatype-strategy/JSONDataTypeStrategy"), exports);
|
|
28
|
+
__exportStar(require("./datatype-strategy/JSONStreamTransformer"), exports);
|
|
26
29
|
__exportStar(require("./datatype-strategy/XMLDataTypeStrategy"), exports);
|
|
27
30
|
__exportStar(require("./protocol-strategy/FTPProtocolStrategy"), exports);
|
|
31
|
+
__exportStar(require("./protocol-strategy/HTTPRequestProtocolStrategyStreamed"), exports);
|
|
28
32
|
__exportStar(require("./protocol-strategy/GoogleCloudStorageProtocolStrategy"), exports);
|
|
29
33
|
__exportStar(require("./protocol-strategy/HTTPFetchProtocolStrategy"), exports);
|
|
30
34
|
__exportStar(require("./protocol-strategy/PaginatedHTTPProtocolStrategy"), exports);
|
|
31
35
|
__exportStar(require("./protocol-strategy/IProtocolStrategy"), exports);
|
|
36
|
+
__exportStar(require("./protocol-strategy/IStreamingProtocolStrategy"), exports);
|
|
32
37
|
__exportStar(require("./protocol-strategy/PostgresProtocolStrategy"), exports);
|
|
33
38
|
__exportStar(require("./protocol-strategy/PostgresProtocolStrategyStreamed"), exports);
|
|
34
39
|
__exportStar(require("./protocol-strategy/ProtocolStrategy"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,0EAAwD;AACxD,0EAAwD;AACxD,yFAAuE;AACvE,gFAA8D;AAC9D,oFAAkE;AAClE,wEAAsD;AACtD,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,kDAAgC;AAChC,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,+DAA6C;AAC7C,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,4EAA0D;AAC1D,0EAAwD;AACxD,0EAAwD;AACxD,0FAAwE;AACxE,yFAAuE;AACvE,gFAA8D;AAC9D,oFAAkE;AAClE,wEAAsD;AACtD,iFAA+D;AAC/D,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { IProtocolStrategy } from "./IProtocolStrategy";
|
|
2
|
+
import { Readable } from "node:stream";
|
|
2
3
|
import { DataSourceStream } from "../DataSourceStream";
|
|
3
4
|
import { HTTPRequestProtocolStrategy } from "./HTTPRequestProtocolStrategy";
|
|
4
|
-
|
|
5
|
+
import { IStreamingProtocolStrategy } from "./IStreamingProtocolStrategy";
|
|
6
|
+
export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy, IStreamingProtocolStrategy {
|
|
5
7
|
private streamTransform;
|
|
6
8
|
/**
|
|
7
9
|
* @param streamTransform data will be piped to this stream if provided
|
|
8
10
|
*/
|
|
9
11
|
setStreamTransformer: (streamTransform: any) => HTTPRequestProtocolStrategyStreamed;
|
|
10
12
|
getData(): Promise<DataSourceStream>;
|
|
13
|
+
getBodyStream(): Promise<Readable>;
|
|
11
14
|
getRawData: () => Promise<any>;
|
|
12
15
|
}
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.HTTPRequestProtocolStrategyStreamed = void 0;
|
|
4
4
|
const errors_1 = require("@golemio/errors");
|
|
5
|
+
const node_zlib_1 = require("node:zlib");
|
|
5
6
|
const DataSourceStream_1 = require("../DataSourceStream");
|
|
6
7
|
const HTTPRequestProtocolStrategy_1 = require("./HTTPRequestProtocolStrategy");
|
|
7
8
|
class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.HTTPRequestProtocolStrategy {
|
|
@@ -16,7 +17,8 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
|
|
|
16
17
|
};
|
|
17
18
|
this.getRawData = async () => {
|
|
18
19
|
if (this.connectionSettings.compression) {
|
|
19
|
-
throw new errors_1.GeneralError("
|
|
20
|
+
throw new errors_1.GeneralError("Compressed resources are not supported in the legacy DataSourceStream path;" +
|
|
21
|
+
"use DataSourceStreamedPipeline instead", this.constructor.name);
|
|
20
22
|
}
|
|
21
23
|
try {
|
|
22
24
|
const result = await this.getResponseInternal();
|
|
@@ -97,6 +99,33 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
|
|
|
97
99
|
}
|
|
98
100
|
return outStream;
|
|
99
101
|
}
|
|
102
|
+
async getBodyStream() {
|
|
103
|
+
try {
|
|
104
|
+
const result = await this.getResponseInternal();
|
|
105
|
+
if (result.statusCode < 200 || result.statusCode >= 300) {
|
|
106
|
+
result.body?.resume?.(); // drain to avoid connection leak (some statuses have no body)
|
|
107
|
+
throw new errors_1.GeneralError(`Unexpected HTTP status ${result.statusCode}`, this.constructor.name);
|
|
108
|
+
}
|
|
109
|
+
let body = result.body;
|
|
110
|
+
if (this.connectionSettings.compression === "gzip") {
|
|
111
|
+
const encoding = String(result.headers?.["content-encoding"] ?? "").toLowerCase();
|
|
112
|
+
if (encoding !== "gzip") {
|
|
113
|
+
body?.resume?.();
|
|
114
|
+
throw new errors_1.GeneralError(`Expected gzip-encoded response but server returned Content-Encoding: "${encoding || "<none>"}"`, this.constructor.name);
|
|
115
|
+
}
|
|
116
|
+
body = body.pipe((0, node_zlib_1.createGunzip)());
|
|
117
|
+
}
|
|
118
|
+
return this.teeRawDataToStorage(body, {
|
|
119
|
+
statusCode: result.statusCode,
|
|
120
|
+
headers: result.headers,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
if (err instanceof errors_1.GeneralError)
|
|
125
|
+
throw err;
|
|
126
|
+
throw new errors_1.GeneralError("Error while getting body stream from server.", this.constructor.name, err);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
100
129
|
}
|
|
101
130
|
exports.HTTPRequestProtocolStrategyStreamed = HTTPRequestProtocolStrategyStreamed;
|
|
102
131
|
//# sourceMappingURL=HTTPRequestProtocolStrategyStreamed.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;
|
|
1
|
+
{"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;AAErE,yCAAyC;AACzC,0DAAuD;AACvD,+EAA4E;AAG5E,MAAa,mCACT,SAAQ,yDAA2B;IADvC;;QAMI;;WAEG;QACI,yBAAoB,GAAG,CAAC,eAAoB,EAAuC,EAAE;YACxF,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAuGK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,CAAC;gBACtC,MAAM,IAAI,qBAAY,CAClB,6EAA6E;oBACzE,wCAAwC,EAC5C,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;YACN,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAEhD,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,IAAI;iBACpB,CAAC;YACN,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAChG,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAxHU,KAAK,CAAC,OAAO;QAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,mCAAgB,CAAC;YACnC,UAAU,EAAE,IAAI;YAChB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC5C,IAAI,KAAK,YAAY,6BAAoB,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACJ,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,6BAA6B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3G,CAAC;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE,CAAC;oBACxC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC9C,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC9G,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACJ,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,aAAa,EAAE,CAAC;oBACjB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,uCAAuC;gBACvC,IAAI,CAAC,aAAa,EAAE,CAAC;oBACjB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;gBACzB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACJ,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBACnG,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,aAAa;QACtB,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEhD,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,8DAA8D;gBACvF,MAAM,IAAI,qBAAY,CAAC,0BAA0B,MAAM,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACjG,CAAC;YAED,IAAI,IAAI,GAAG,MAAM,CAAC,IAA2B,CAAC;YAE9C,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;gBACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBAClF,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;oBACtB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAY,CAClB,yEAAyE,QAAQ,IAAI,QAAQ,GAAG,EAChG,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;gBACN,CAAC;gBACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC;YACrC,CAAC;YAED,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE;gBAClC,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;aAC1B,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,8CAA8C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACvG,CAAC;IACL,CAAC;CAqBJ;AAtID,kFAsIC"}
|
package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IStreamingProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.ts"],"names":[],"mappings":""}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
1
2
|
import { DataSourceStream } from "../DataSourceStream";
|
|
2
3
|
import { IProtocolStrategy, ProtocolStrategySettings } from "./IProtocolStrategy";
|
|
3
4
|
export declare class ProtocolStrategy implements IProtocolStrategy {
|
|
@@ -18,6 +19,7 @@ export declare class ProtocolStrategy implements IProtocolStrategy {
|
|
|
18
19
|
}>;
|
|
19
20
|
getLastModified: () => Promise<string | null>;
|
|
20
21
|
protected getRawData(...args: any): Promise<any | DataSourceStream>;
|
|
22
|
+
protected teeRawDataToStorage(bodyStream: Readable, meta?: unknown): Readable;
|
|
21
23
|
private prepareRawData;
|
|
22
24
|
private pushRawDataToS3Stream;
|
|
23
25
|
}
|
|
@@ -5,7 +5,8 @@ const CoreToken_1 = require("../../../helpers/ioc/CoreToken");
|
|
|
5
5
|
const ioc_1 = require("../../ioc");
|
|
6
6
|
const Di_1 = require("../../ioc/Di");
|
|
7
7
|
const errors_1 = require("@golemio/errors");
|
|
8
|
-
const
|
|
8
|
+
const node_stream_1 = require("node:stream");
|
|
9
|
+
const promises_1 = require("node:stream/promises");
|
|
9
10
|
const DataSourceStream_1 = require("../DataSourceStream");
|
|
10
11
|
class ProtocolStrategy {
|
|
11
12
|
constructor(settings) {
|
|
@@ -89,9 +90,9 @@ class ProtocolStrategy {
|
|
|
89
90
|
let dataSaveCalled = false;
|
|
90
91
|
let protocolDataStream;
|
|
91
92
|
// bit paranoid but ...
|
|
92
|
-
if (rawData && (rawData instanceof
|
|
93
|
+
if (rawData && (rawData instanceof node_stream_1.Readable || rawData.constructor.name === "Request") && rawData.on) {
|
|
93
94
|
protocolDataStream = rawData;
|
|
94
|
-
const S3Stream = new
|
|
95
|
+
const S3Stream = new node_stream_1.Readable({
|
|
95
96
|
objectMode: false,
|
|
96
97
|
read: () => {
|
|
97
98
|
return;
|
|
@@ -145,6 +146,44 @@ class ProtocolStrategy {
|
|
|
145
146
|
async getRawData(...args) {
|
|
146
147
|
throw new Error("getRawData: Method not implemented.");
|
|
147
148
|
}
|
|
149
|
+
teeRawDataToStorage(bodyStream, meta = {}) {
|
|
150
|
+
if (!this.storageEnabled || !this.saveRawDataWhitelist[this.caller]) {
|
|
151
|
+
return bodyStream;
|
|
152
|
+
}
|
|
153
|
+
const S3Stream = new node_stream_1.Readable({
|
|
154
|
+
objectMode: false,
|
|
155
|
+
read: () => {
|
|
156
|
+
return;
|
|
157
|
+
},
|
|
158
|
+
});
|
|
159
|
+
const consumer = new node_stream_1.PassThrough();
|
|
160
|
+
let consumerErrorFired = false;
|
|
161
|
+
(0, promises_1.pipeline)(bodyStream, consumer).catch((err) => {
|
|
162
|
+
if (!consumerErrorFired) {
|
|
163
|
+
this.logger.error(`Body stream source failed for ${this.caller}`, err);
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
let saveCalled = false;
|
|
167
|
+
consumer.on("data", async (chunk) => {
|
|
168
|
+
await this.pushRawDataToS3Stream(chunk, S3Stream);
|
|
169
|
+
if (!saveCalled) {
|
|
170
|
+
this.rawDataStorage.save(S3Stream, meta, this.caller);
|
|
171
|
+
saveCalled = true;
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
consumer.on("end", () => {
|
|
175
|
+
S3Stream.push(null);
|
|
176
|
+
});
|
|
177
|
+
consumer.on("error", (err) => {
|
|
178
|
+
consumerErrorFired = true;
|
|
179
|
+
this.logger.warn(`Raw data save to storage interrupted for ${this.caller}`, err);
|
|
180
|
+
S3Stream.push(null);
|
|
181
|
+
});
|
|
182
|
+
S3Stream.on("error", (err) => {
|
|
183
|
+
this.logger.error(`Error on saving raw data for ${this.caller}`, err);
|
|
184
|
+
});
|
|
185
|
+
return consumer;
|
|
186
|
+
}
|
|
148
187
|
}
|
|
149
188
|
exports.ProtocolStrategy = ProtocolStrategy;
|
|
150
189
|
//# sourceMappingURL=ProtocolStrategy.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;AACA,8DAAmD;AAInD,mCAAyC;AACzC,qCAAwD;AACxD,4CAAqE;AACrE,
|
|
1
|
+
{"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;AACA,8DAAmD;AAInD,mCAAyC;AACzC,qCAAwD;AACxD,4CAAqE;AACrE,6CAAoD;AACpD,mDAAgD;AAChD,0DAAuD;AAGvD,MAAa,gBAAgB;IASzB,YAAY,QAAa;QANjB,WAAM,GAAG,SAAS,CAAC;QAgBpB,kBAAa,GAAG,CAAC,SAAiB,oBAAoB,EAAE,EAAE;YAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,iBAAiB;QAC3C,CAAC,CAAC;QAMK,0BAAqB,GAAG,GAA6B,EAAE;YAC1D,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC,CAAC;QA8EK,oBAAe,GAAG,KAAK,IAA4B,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC,CAAC;QAqDM,mBAAc,GAAG,KAAK,EAAE,IAAS,EAA0B,EAAE;YACjE,IAAI,OAAO,GAAkB,IAAI,CAAC;YAElC,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;gBACvB,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;gBACrC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;gBACvD,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBAAC,MAAM,CAAC;oBACL,IAAI,CAAC;gBACT,CAAC;YACL,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACxD,CAAC;gBAAC,MAAM,CAAC;oBACL,IAAI,CAAC;gBACT,CAAC;YACL,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACtC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,CAAC;YAED,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC;QAEM,0BAAqB,GAAG,KAAK,EAAE,IAAS,EAAE,QAAkB,EAAiB,EAAE;YACnF,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACzC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;wBAC9B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACxB,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,CAAC,MAAM;;qBAE3D,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QA3ME,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,MAAM,YAAY,GAAG,+BAA0B,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QAC/F,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC;QACvF,MAAM,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,MAAM,CAAC,CAAC;QACzF,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC,cAAc,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,cAAc,CAAC,CAAC;IAC5G,CAAC;IAMM,qBAAqB,CAAC,QAAa;QACtC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAMM,KAAK,CAAC,OAAO,CAAC,GAAG,IAAW;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,GAAG,IAAe;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,kBAAuB,CAAC;QAE5B,uBAAuB;QACvB,IAAI,OAAO,IAAI,CAAC,OAAO,YAAY,sBAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACnG,kBAAkB,GAAG,OAAO,CAAC;YAE7B,MAAM,QAAQ,GAAG,IAAI,sBAAQ,CAAC;gBAC1B,UAAU,EAAE,KAAK;gBACjB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC;gBACpC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,sFAAsF;YACtF,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAEjD,oGAAoG;gBACpG,IAAI,CAAC,cAAc,EAAE,CAAC;oBAClB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtD,cAAc,GAAG,IAAI,CAAC;gBAC1B,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC1C,IAAI,GAAG,YAAY,6BAAoB,EAAE,CAAC;oBACtC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;qBAAM,CAAC;oBACJ,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;gBACxG,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC9B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,OAAO;YACX,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAQ,EAAE,EAAE;gBAC3C,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;gBACjC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAChD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAChF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7C,CAAC;IACL,CAAC;IAMS,KAAK,CAAC,UAAU,CAAC,GAAG,IAAS;QACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;IAES,mBAAmB,CAAC,UAAoB,EAAE,OAAgB,EAAE;QAClE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAClE,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,sBAAQ,CAAC;YAC1B,UAAU,EAAE,KAAK;YACjB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,yBAAW,EAAE,CAAC;QAEnC,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAA,mBAAQ,EAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACzC,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YAC3E,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAU,EAAE,EAAE;YACrC,MAAM,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClD,IAAI,CAAC,UAAU,EAAE,CAAC;gBACd,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtD,UAAU,GAAG,IAAI,CAAC;YACtB,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACpB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,kBAAkB,GAAG,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YACjF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IACpB,CAAC;CAsDJ;AAtND,4CAsNC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/core",
|
|
3
|
-
"version": "4.0.
|
|
3
|
+
"version": "4.1.0-rc.2893266968",
|
|
4
4
|
"description": "Golemio Core Module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
"@types/chai-as-promised": "^8.0.2",
|
|
39
39
|
"@types/content-type": "^1.1.3",
|
|
40
40
|
"@types/debug": "0.0.31",
|
|
41
|
+
"@types/jsonstream": "^0.8.33",
|
|
41
42
|
"@types/mocha": "^10.0.10",
|
|
42
43
|
"@types/node": "^24.12.4",
|
|
43
44
|
"@types/sinon": "^21.0.0",
|
|
@@ -77,7 +78,7 @@
|
|
|
77
78
|
"@google-cloud/storage": "^7.19.0",
|
|
78
79
|
"@opensearch-project/opensearch": "^3.6.0",
|
|
79
80
|
"@opentelemetry/api": "^1.9.0",
|
|
80
|
-
"@opentelemetry/exporter-trace-otlp-grpc": "^0.
|
|
81
|
+
"@opentelemetry/exporter-trace-otlp-grpc": "^0.214.0",
|
|
81
82
|
"@opentelemetry/instrumentation-amqplib": "^0.50.0",
|
|
82
83
|
"@opentelemetry/instrumentation-aws-sdk": "^0.56.0",
|
|
83
84
|
"@opentelemetry/instrumentation-express": "^0.52.0",
|
|
@@ -91,7 +92,7 @@
|
|
|
91
92
|
"@types/geojson": "^7946.0.10",
|
|
92
93
|
"@types/luxon": "^2.3.1",
|
|
93
94
|
"@types/qs": "^6.9.18",
|
|
94
|
-
"ajv": "^8.
|
|
95
|
+
"ajv": "^8.20.0",
|
|
95
96
|
"amqplib": "^0.10.8",
|
|
96
97
|
"basic-ftp": "^6.0.1",
|
|
97
98
|
"class-transformer": "^0.5.1",
|
|
@@ -106,7 +107,8 @@
|
|
|
106
107
|
"html-entities": "^2.4.0",
|
|
107
108
|
"iconv-lite": "^0.6.2",
|
|
108
109
|
"ioredis": "^5.4.1",
|
|
109
|
-
"
|
|
110
|
+
"JSONStream": "^1.3.5",
|
|
111
|
+
"lightship": "^9.0.4",
|
|
110
112
|
"luxon": "^2.5.2",
|
|
111
113
|
"moment-timezone": "^0.5.26",
|
|
112
114
|
"opentelemetry-instrumentation-sequelize": "^0.41.0",
|
|
@@ -128,6 +130,7 @@
|
|
|
128
130
|
"yauzl": "^2.10.0"
|
|
129
131
|
},
|
|
130
132
|
"overrides": {
|
|
133
|
+
"fast-uri": "^3.1.8",
|
|
131
134
|
"lightship": {
|
|
132
135
|
"fastify": "^5.8.5"
|
|
133
136
|
}
|