@rainbow-o23/n3 1.0.56 → 1.0.58-alpha.2

package/README.md

@@ -492,9 +492,16 @@ Array<TypeOrmEntityToLoad>;
 
 ##### Environment Parameters
 
-| Name                    | Type   | Default Value | Comments    |
-|-------------------------|--------|---------------|-------------|
-| `typeorm.DB.fetch.size` | number | 20            | Fetch size. |
+| Name                              | Type    | Default Value | Comments                               |
+|-----------------------------------|---------|---------------|----------------------------------------|
+| `typeorm.DB.fetch.size`           | number  | 20            | Fetch size.                            |
+| `typeorm.DB.stream.pause.enabled` | boolean | false         | Pause and resume result stream or not. |
+
+##### Constructor Parameters
+
+| Name               | Type    | Default Value | Comments                               |
+|--------------------|---------|---------------|----------------------------------------|
+| pauseStreamEnabled | boolean |               | Pause and resume result stream or not. |
 
 ##### Request and Response
 
@@ -585,13 +592,17 @@ step set. Additionally, nested transactions are also supported, which means Tran
 
 #### Environment Parameters
 
-| Name                                | Type   | Default Value | Comments                                                                                                                          |
-|-------------------------------------|--------|---------------|-----------------------------------------------------------------------------------------------------------------------------------|
-| `endpoints.SYSTEM.ENDPOINT.url`     | string |               | Endpoint URL.                                                                                                                     |
-| `endpoints.SYSTEM.ENDPOINT.headers` | string |               | Endpoint request headers, use global headers if this parameter doesn't present.<br>Format follows `name=value[;name=value[...]]`. |
-| `endpoints.SYSTEM.global.headers`   | string |               | Endpoint system global request headers.<br>Format follows `name=value[;name=value[...]]`.                                         |
-| `endpoints.SYSTEM.ENDPOINT.timeout` | number |               | Endpoint request timeout, in seconds, use global timeout if this parameter doesn't present.                                       |
-| `endpoints.SYSTEM.global.timeout`   | number | -1            | Endpoint system global timeout, in seconds, `-1` represents no timeout.                                                           |
+| Name                                                    | Type   | Default Value | Comments                                                                                                                                                                    |
+|---------------------------------------------------------|--------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `endpoints.SYSTEM.ENDPOINT.url`                         | string |               | Endpoint URL.                                                                                                                                                               |
+| `endpoints.SYSTEM.ENDPOINT.headers`                     | string |               | Endpoint request headers, use global headers if this parameter doesn't present.<br>Format follows `name=value[;name=value[...]]`.                                           |
+| `endpoints.SYSTEM.global.headers`                       | string |               | Endpoint system global request headers.<br>Format follows `name=value[;name=value[...]]`.                                                                                   |
+| `endpoints.SYSTEM.ENDPOINT.headers.transparent`         | string |               | Endpoint request transparent-passed headers names, use global headers if this parameter doesn't present.<br>Format follows `name1[;name2[...]]`.                            |
+| `endpoints.SYSTEM.global.headers.transparent`           | string |               | Endpoint system global request transparent-passed headers names.<br>Format follows `name1[;name2[...]]`.                                                                    |
+| `endpoints.SYSTEM.ENDPOINT.headers.transparent.omitted` | string |               | Endpoint request headers names which omitted from transparent-passed headers, use global headers if this parameter doesn't present.<br>Format follows `name1[;name2[...]]`. |
+| `endpoints.SYSTEM.global.headers.transparent.omitted`   | string |               | Endpoint system global request headers names which omitted from transparent-passed headers.<br>Format follows `name1[;name2[...]]`.                                         |
+| `endpoints.SYSTEM.ENDPOINT.timeout`                     | number |               | Endpoint request timeout, in seconds, use global timeout if this parameter doesn't present.                                                                                 |
+| `endpoints.SYSTEM.global.timeout`                       | number | -1            | Endpoint system global timeout, in seconds, `-1` represents no timeout.                                                                                                     |
 
 `SYSTEM` represents endpoint system, `ENDPOINT` represents endpoint url. For example:
 
@@ -604,18 +615,60 @@ CFG_ENDPOINTS_ORDER_PAYMENT_URL=https://order.com/payment
 
 #### Constructor Parameters
 
-| Name                 | Type                                                                                                   | Default Value | Comments                                                                                                             |
-|----------------------|--------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------|
-| endpointSystemCode   | string                                                                                                 |               | Endpoint system code.                                                                                                |
-| endpointName         | string                                                                                                 |               | Endpoint name.                                                                                                       |
-| urlGenerate          | ScriptFuncOrBody\<HttpGenerateUrl>                                                                     |               | Endpoint url generator, `$endpointUrl`.                                                                              |
-| method               | string                                                                                                 |               | Http method, default `post`.                                                                                         |
-| timeout              | number                                                                                                 |               | Endpoint timeout, in seconds.                                                                                        |
-| headersGenerate      | ScriptFuncOrBody\<HttpGenerateHeaders>                                                                 |               | Endpoint request headers generator.                                                                                  |
-| bodyUsed             | boolean                                                                                                |               | Send request with body or not, or automatically disregards the body when sending a `get` request when not specified. |
-| bodyGenerate         | ScriptFuncOrBody\<HttpGenerateBody>                                                                    |               | Endpoint request body generator.                                                                                     |
-| responseGenerate     | ScriptFuncOrBody\<HttpGenerateResponse>                                                                |               | Endpoint response body generator, `$response`.                                                                       |
-| responseErrorHandles | ScriptFuncOrBody\<HttpHandleError><br>or<br>{[key: HttpErrorCode]: ScriptFuncOrBody\<HttpHandleError>} |               | Endpoint response error handlers.                                                                                    |
+| Name                          | Type                                                                                                   | Default Value | Comments                                                                                                             |
+|-------------------------------|--------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------|
+| endpointSystemCode            | string                                                                                                 |               | Endpoint system code.                                                                                                |
+| endpointName                  | string                                                                                                 |               | Endpoint name.                                                                                                       |
+| urlGenerate                   | ScriptFuncOrBody\<HttpGenerateUrl>                                                                     |               | Endpoint url generator, `$endpointUrl`.                                                                              |
+| method                        | string                                                                                                 |               | Http method, default `post`.                                                                                         |
+| timeout                       | number                                                                                                 |               | Endpoint timeout, in seconds.                                                                                        |
+| transparentHeaderNames        | Array<string>                                                                                          |               | Transparent-passed request headers names.                                                                            |
+| omittedTransparentHeaderNames | Array<string>                                                                                          |               | Omitted names of transparently-passed request headers.                                                               |
+| headersGenerate               | ScriptFuncOrBody\<HttpGenerateHeaders>                                                                 |               | Endpoint request headers generator.                                                                                  |
+| bodyUsed                      | boolean                                                                                                |               | Send request with body or not, or automatically disregards the body when sending a `get` request when not specified. |
+| bodyGenerate                  | ScriptFuncOrBody\<HttpGenerateBody>                                                                    |               | Endpoint request body generator.                                                                                     |
+| responseGenerate              | ScriptFuncOrBody\<HttpGenerateResponse>                                                                |               | Endpoint response body generator, `$response`.                                                                       |
+| responseErrorHandles          | ScriptFuncOrBody\<HttpHandleError><br>or<br>{[key: HttpErrorCode]: ScriptFuncOrBody\<HttpHandleError>} |               | Endpoint response error handlers.                                                                                    |
+
+- `transparentHeaderNames` and `omittedTransparentHeaderNames`:
+  Use `transparentHeaderNames` to specify the names of request headers whose values need to be transparently passed from the input
+  parameters to the downstream service. Separate the names with ';'. The names support using `.` for connection so that values from
+  multi-level objects can be directly retrieved. For example, `account.name` will retrieve the value of the `name` property from the
+  `account` property of the input object. When writing the values into the header values, the following rules apply:
+	- If the value is an array, use `, ` to connect the elements. `null` and empty strings will be filtered out.
+	- If the value is an object, use the object's keys to generate multiple headers. `null` and empty strings will be filtered out.
+	- For other values, convert them to strings. `null` and empty strings will be filtered out.
+	- Note that an empty string does not include blank strings, and no automatic trimming will be performed.
+
+  If the `transparentHeaderNames` at the step level is not defined, use the definition in
+  `endpoints.SYSTEM.ENDPOINT.headers.transparent`. If it is also not defined at the endpoint level, use the definition in
+  `endpoints.SYSTEM.global.headers.transparent`.
+
+  After obtaining the transparently passed request headers, check the definition of `omittedTransparentHeaderNames`. If it is defined,
+  remove the corresponding headers from the headers. `omittedTransparentHeaderNames` is case-insensitive. Similarly, if the
+  `omittedTransparentHeaderNames` at the step level is not defined, use the definition in
+  `endpoints.SYSTEM.ENDPOINT.headers.transparent.omitted`. If it is also not defined at the endpoint level, use the definition in
+  `endpoints.SYSTEM.global.headers.transparent.omitted`.
+
+  For example:
+  If the input data contains `{account: {name: 'John', token: '******'}}` and `transparentHeaderNames` is defined as `account`, then two
+  transparently passed headers will be obtained: `name=John` and `token=******`. At this time, if `omittedTransparentHeaderNames` is defined
+  as `name`, the headers that will be finally transparently passed to the downstream service are `token=******`, and `name` will be ignored.
+
+## Request headers
+
+There are three ways to transmit request headers to downstream services. In order of priority from high to low, they are `headersGenerate`,
+`headers.transparent`, and `headers`. If a header appears in a higher-priority method, the header with the same name generated by a
+lower-priority method will be ignored. Note that the matching is case-sensitive.
+
+Normally, if you need to transparently pass the request headers from the client to the downstream service, you should use `headers: true` in
+the pipeline definition. Then you can directly use `transparentHeaderNames: headers` to obtain all the request headers, and then use
+`omittedTransparentHeaderNames` for necessary filtering.
+
+> It should be noted that since the fetch step initiates a new request to the downstream service, its request structure and data will be
+> modified or reset according to requirements. Therefore, even if you need to transparently pass the request headers, some of the headers
+> are still not applicable. So by default, the two headers `content-encoding` and `content-length` will be filtered out. No matter how the
+> request headers are generated in the above process, these two headers are always automatically generated by `node-fetch`.
 
 ## Installation
 

package/index.cjs

@@ -1281,6 +1281,8 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     _endpointTimeout;
     _urlGenerateSnippet;
     _urlGenerateFunc;
+    _transparentHeaderNames;
+    _omittedTransparentHeaderNames;
     _headersGenerateSnippet;
     _headersGenerateFunc;
     _bodyUsed;
@@ -1311,6 +1313,10 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
                 throw e;
             }
         });
+        this._transparentHeaderNames = options.transparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent`)));
+        this._omittedTransparentHeaderNames = options.omittedTransparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent.omitted`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent.omitted`)));
         this._headersGenerateSnippet = options.headersGenerate;
         this._headersGenerateFunc = Utils.createAsyncFunction(this.getHeadersGenerateSnippet(), {
             createDefault: () => async (_$factor, _$request, _$helpers, _$) => (void 0),
@@ -1334,8 +1340,15 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
         });
         this._responseGenerateSnippet = options.responseGenerate;
         this._responseGenerateFunc = Utils.createAsyncFunction(this.getResponseGenerateSnippet(), {
-            createDefault: () => async ($response, _$factor, _$request, _$helpers, _$) => {
-                return await $response.json();
+            createDefault: () => async ($response, _$factor, _$request, $helpers, _$) => {
+                const contentEncoding = $response.headers?.get('content-encoding');
+                if (contentEncoding === 'zstd') {
+                    const buffer = await $response.buffer();
+                    return JSON.parse(await $helpers.$zstd(buffer));
+                }
+                else {
+                    return await $response.json();
+                }
             },
             getVariableNames: () => this.getResponseGenerateVariableName(),
             error: (e) => {
@@ -1427,6 +1440,20 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     getUrlGenerateVariableName() {
         return ['$endpointUrl', '$factor', '$request', ...this.getHelpersVariableNames()];
     }
+    getTransparentHeaderNames() {
+        return this._transparentHeaderNames ?? [];
+    }
+    getOmittedTransparentHeaderNames() {
+        return this._omittedTransparentHeaderNames ?? [];
+    }
+    generateTransparentHeaderNames(headerNames, base) {
+        return [
+            ...(base ?? []),
+            ...new Set(`${headerNames || ''}`.split(';')
+                .map(x => x.trim())
+                .filter(x => x.length !== 0))
+        ];
+    }
     getHeadersGenerateSnippet() {
         return this._headersGenerateSnippet;
     }
@@ -1458,7 +1485,37 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             url = await this._urlGenerateFunc(this.getEndpointUrl(), data, request, $helpers, $helpers);
             const method = this.getEndpointMethod();
             const staticHeaders = this.getEndpointHeaders() ?? {};
-            const headers = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
+            const transparentHeaders = (this.getTransparentHeaderNames() ?? []).reduce((headers, name) => {
+                const value = Utils.getValue(data, name);
+                if (value == null) {
+                }
+                else if (Array.isArray(value)) {
+                    const headerValue = value.filter(v => v != null && `${v}`.length !== 0).join(', ');
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                else if (typeof value === 'object') {
+                    Object.keys(value).forEach(key => {
+                        const headerValue = value[key];
+                        if (headerValue != null) {
+                            const s = `${headerValue}`;
+                            if (s.length !== 0) {
+                                headers[key] = s;
+                            }
+                        }
+                    });
+                }
+                else {
+                    const headerValue = `${value}`;
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                return headers;
+            }, {});
+            (this.getOmittedTransparentHeaderNames() ?? []).forEach(name => delete transparentHeaders[name]);
+            const generatedHeaders = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
             let body;
             const bodyUsed = this.isBodyUsed();
             if (bodyUsed === true || (bodyUsed == null && method !== 'get')) {
@@ -1470,8 +1527,14 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             if (body != null && typeof body !== 'string') {
                 body = JSON.stringify(body);
             }
+            const headers = { ...staticHeaders, ...transparentHeaders, ...generatedHeaders };
+            Object.keys(headers).filter(name => {
+                return ['content-encoding', 'content-length'].includes(name.toLowerCase());
+            }).forEach(name => {
+                delete headers[name];
+            });
             const response = await fetch(url, {
-                method, headers: { ...staticHeaders, ...headers }, body,
+                method, headers, body,
                 signal: this.needTimeout() ? (() => {
                     const controller = new AbortController();
                     setTimeout(() => controller.abort(), this.getEndpointTimeout());
@@ -2223,6 +2286,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     _streamToSnippet;
     _streamToFunc;
     _stepBuilders;
+    _pauseStreamEnabled;
     constructor(options) {
         super(options);
         const config = this.getConfig();
@@ -2237,6 +2301,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
             }
         });
         this._stepBuilders = options.steps;
+        this._pauseStreamEnabled = options.pauseStreamEnabled ?? config.getBoolean(`typeorm.${this.getDataSourceName()}.stream.pause.enabled`, false);
     }
     getFetchSize() {
         return this._fetchSize;
@@ -2250,6 +2315,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     getStepBuilders() {
         return this._stepBuilders ?? [];
     }
+    isPauseStreamEnabled() {
+        return this._pauseStreamEnabled;
+    }
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.params);
         return await this.autoTrans(async (runner) => {
@@ -2312,7 +2380,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                     reject(e);
                 });
                 readable.on('data', async (data) => {
-                    readable.pause();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.pause();
+                    }
                     rows.push(data);
                     await pipe({
                         resolve, reject: async (e) => {
@@ -2320,7 +2390,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                             reject(e);
                         }, end: false
                     });
-                    readable.resume();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.resume();
+                    }
                 });
             };
             return new Promise((resolve, reject) => read({ resolve, reject }));

package/index.js

@@ -1279,6 +1279,8 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     _endpointTimeout;
     _urlGenerateSnippet;
     _urlGenerateFunc;
+    _transparentHeaderNames;
+    _omittedTransparentHeaderNames;
     _headersGenerateSnippet;
     _headersGenerateFunc;
     _bodyUsed;
@@ -1309,6 +1311,10 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
                 throw e;
             }
         });
+        this._transparentHeaderNames = options.transparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent`)));
+        this._omittedTransparentHeaderNames = options.omittedTransparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent.omitted`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent.omitted`)));
         this._headersGenerateSnippet = options.headersGenerate;
         this._headersGenerateFunc = Utils.createAsyncFunction(this.getHeadersGenerateSnippet(), {
             createDefault: () => async (_$factor, _$request, _$helpers, _$) => (void 0),
@@ -1332,8 +1338,15 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
         });
         this._responseGenerateSnippet = options.responseGenerate;
         this._responseGenerateFunc = Utils.createAsyncFunction(this.getResponseGenerateSnippet(), {
-            createDefault: () => async ($response, _$factor, _$request, _$helpers, _$) => {
-                return await $response.json();
+            createDefault: () => async ($response, _$factor, _$request, $helpers, _$) => {
+                const contentEncoding = $response.headers?.get('content-encoding');
+                if (contentEncoding === 'zstd') {
+                    const buffer = await $response.buffer();
+                    return JSON.parse(await $helpers.$zstd(buffer));
+                }
+                else {
+                    return await $response.json();
+                }
             },
             getVariableNames: () => this.getResponseGenerateVariableName(),
             error: (e) => {
@@ -1425,6 +1438,20 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     getUrlGenerateVariableName() {
         return ['$endpointUrl', '$factor', '$request', ...this.getHelpersVariableNames()];
     }
+    getTransparentHeaderNames() {
+        return this._transparentHeaderNames ?? [];
+    }
+    getOmittedTransparentHeaderNames() {
+        return this._omittedTransparentHeaderNames ?? [];
+    }
+    generateTransparentHeaderNames(headerNames, base) {
+        return [
+            ...(base ?? []),
+            ...new Set(`${headerNames || ''}`.split(';')
+                .map(x => x.trim())
+                .filter(x => x.length !== 0))
+        ];
+    }
     getHeadersGenerateSnippet() {
         return this._headersGenerateSnippet;
     }
@@ -1456,7 +1483,37 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             url = await this._urlGenerateFunc(this.getEndpointUrl(), data, request, $helpers, $helpers);
             const method = this.getEndpointMethod();
             const staticHeaders = this.getEndpointHeaders() ?? {};
-            const headers = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
+            const transparentHeaders = (this.getTransparentHeaderNames() ?? []).reduce((headers, name) => {
+                const value = Utils.getValue(data, name);
+                if (value == null) {
+                }
+                else if (Array.isArray(value)) {
+                    const headerValue = value.filter(v => v != null && `${v}`.length !== 0).join(', ');
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                else if (typeof value === 'object') {
+                    Object.keys(value).forEach(key => {
+                        const headerValue = value[key];
+                        if (headerValue != null) {
+                            const s = `${headerValue}`;
+                            if (s.length !== 0) {
+                                headers[key] = s;
+                            }
+                        }
+                    });
+                }
+                else {
+                    const headerValue = `${value}`;
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                return headers;
+            }, {});
+            (this.getOmittedTransparentHeaderNames() ?? []).forEach(name => delete transparentHeaders[name]);
+            const generatedHeaders = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
             let body;
             const bodyUsed = this.isBodyUsed();
             if (bodyUsed === true || (bodyUsed == null && method !== 'get')) {
@@ -1468,8 +1525,14 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             if (body != null && typeof body !== 'string') {
                 body = JSON.stringify(body);
             }
+            const headers = { ...staticHeaders, ...transparentHeaders, ...generatedHeaders };
+            Object.keys(headers).filter(name => {
+                return ['content-encoding', 'content-length'].includes(name.toLowerCase());
+            }).forEach(name => {
+                delete headers[name];
+            });
             const response = await fetch(url, {
-                method, headers: { ...staticHeaders, ...headers }, body,
+                method, headers, body,
                 signal: this.needTimeout() ? (() => {
                     const controller = new AbortController();
                     setTimeout(() => controller.abort(), this.getEndpointTimeout());
@@ -2221,6 +2284,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     _streamToSnippet;
     _streamToFunc;
     _stepBuilders;
+    _pauseStreamEnabled;
     constructor(options) {
         super(options);
         const config = this.getConfig();
@@ -2235,6 +2299,7 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
             }
         });
         this._stepBuilders = options.steps;
+        this._pauseStreamEnabled = options.pauseStreamEnabled ?? config.getBoolean(`typeorm.${this.getDataSourceName()}.stream.pause.enabled`, false);
     }
     getFetchSize() {
         return this._fetchSize;
@@ -2248,6 +2313,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
     getStepBuilders() {
         return this._stepBuilders ?? [];
     }
+    isPauseStreamEnabled() {
+        return this._pauseStreamEnabled;
+    }
     async doPerform(basis, request) {
         const { sql, params } = this.getSql(basis, basis?.params);
         return await this.autoTrans(async (runner) => {
@@ -2310,7 +2378,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                     reject(e);
                 });
                 readable.on('data', async (data) => {
-                    readable.pause();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.pause();
+                    }
                     rows.push(data);
                     await pipe({
                         resolve, reject: async (e) => {
@@ -2318,7 +2388,9 @@ class TypeOrmLoadManyBySQLUseCursorPipelineStep extends AbstractTypeOrmBySQLPipe
                             reject(e);
                         }, end: false
                     });
-                    readable.resume();
+                    if (this.isPauseStreamEnabled()) {
+                        readable.resume();
+                    }
                 });
             };
             return new Promise((resolve, reject) => read({ resolve, reject }));

package/lib/http/fetch-step.d.ts

@@ -7,6 +7,8 @@ export interface FetchPipelineStepOptions<In = PipelineStepPayload, Out = Pipeli
     urlGenerate?: ScriptFuncOrBody<HttpGenerateUrl<In, InFragment>>;
     method?: string;
     timeout?: number;
+    transparentHeaderNames?: Array<string>;
+    omittedTransparentHeaderNames?: Array<string>;
     headersGenerate?: ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
     bodyUsed?: boolean;
     bodyGenerate?: ScriptFuncOrBody<HttpGenerateBody<In, InFragment, BodyData>>;
@@ -24,6 +26,8 @@ export declare class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineS
     private readonly _endpointTimeout;
     private readonly _urlGenerateSnippet;
     private readonly _urlGenerateFunc;
+    private readonly _transparentHeaderNames;
+    private readonly _omittedTransparentHeaderNames;
     private readonly _headersGenerateSnippet;
     private readonly _headersGenerateFunc;
     private readonly _bodyUsed;
@@ -44,6 +48,9 @@ export declare class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineS
     needTimeout(): boolean;
     getUrlGenerateSnippet(): ScriptFuncOrBody<HttpGenerateUrl<In, InFragment>>;
     protected getUrlGenerateVariableName(): Array<string>;
+    getTransparentHeaderNames(): Array<string>;
+    getOmittedTransparentHeaderNames(): Array<string>;
+    protected generateTransparentHeaderNames(headerNames?: string, base?: Array<string>): Array<string>;
     getHeadersGenerateSnippet(): ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
     protected getHeadersGenerateVariableNames(): Array<string>;
     protected isBodyUsed(): boolean | undefined;

package/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.d.ts

@@ -8,17 +8,20 @@ export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineS
     fetchSize?: number;
     streamTo?: ScriptFuncOrBody<StreamToFunc<In, Item>>;
     steps?: Array<PipelineStepBuilder>;
+    pauseStreamEnabled?: boolean;
 }
 export declare class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload, Out = PipelineStepPayload, InFragment = Undefinable<TypeOrmLoadBasis>, OutFragment = Out, Item = any> extends AbstractTypeOrmBySQLPipelineStep<In, Out, Undefinable<TypeOrmLoadBasis>, OutFragment> {
     private readonly _fetchSize;
     private readonly _streamToSnippet;
     private readonly _streamToFunc;
     private readonly _stepBuilders;
+    private readonly _pauseStreamEnabled;
     constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>);
     protected getFetchSize(): number;
     getStreamToSnippet(): ScriptFuncOrBody<StreamToFunc<In, Item>>;
     protected generateVariableNames(): Array<string>;
     protected getStepBuilders(): Array<PipelineStepBuilder>;
+    protected isPauseStreamEnabled(): boolean;
     protected doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>>;
     protected getFetchDataVariableName(): string;
     protected getRequestVariableName(): string;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.56",
+	"version": "1.0.58-alpha.2",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,7 +21,7 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.56",
+		"@rainbow-o23/n1": "1.0.58-alpha.2",
 		"node-fetch": "2.6.7",
 		"typeorm": "^0.3.20",
 		"typescript": "5.5.4"
@@ -30,6 +30,7 @@
 		"@babel/core": "^7.23.9",
 		"@babel/preset-env": "^7.23.9",
 		"@babel/preset-typescript": "^7.23.3",
+		"@oneidentity/zstd-js": "^1.0.3",
 		"@rollup/plugin-babel": "^6.0.4",
 		"@rollup/plugin-eslint": "^9.0.3",
 		"@types/better-sqlite3": "^7.6.6",

package/src/lib/http/fetch-step.ts

@@ -23,6 +23,8 @@ export interface FetchPipelineStepOptions<In = PipelineStepPayload, Out = Pipeli
 	method?: string;
 	/** on seconds */
 	timeout?: number;
+	transparentHeaderNames?: Array<string>;
+	omittedTransparentHeaderNames?: Array<string>;
 	headersGenerate?: ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
 	bodyUsed?: boolean;
 	bodyGenerate?: ScriptFuncOrBody<HttpGenerateBody<In, InFragment, BodyData>>;
@@ -42,6 +44,8 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 	private readonly _endpointTimeout: number;
 	private readonly _urlGenerateSnippet: ScriptFuncOrBody<HttpGenerateUrl<In, InFragment>>;
 	private readonly _urlGenerateFunc: HttpGenerateUrl<In, InFragment>;
+	private readonly _transparentHeaderNames: Array<string>;
+	private readonly _omittedTransparentHeaderNames: Array<string>;
 	private readonly _headersGenerateSnippet: ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
 	private readonly _headersGenerateFunc: HttpGenerateHeaders<In, InFragment>;
 	private readonly _bodyUsed: boolean;
@@ -78,6 +82,14 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 				throw e;
 			}
 		});
+		this._transparentHeaderNames = options.transparentHeaderNames
+			?? this.generateTransparentHeaderNames(
+				config.getString(`endpoints.${endpointKey}.headers.transparent`),
+				this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent`)));
+		this._omittedTransparentHeaderNames = options.omittedTransparentHeaderNames
+			?? this.generateTransparentHeaderNames(
+				config.getString(`endpoints.${endpointKey}.headers.transparent.omitted`),
+				this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent.omitted`)));
 		this._headersGenerateSnippet = options.headersGenerate;
 		this._headersGenerateFunc = Utils.createAsyncFunction(this.getHeadersGenerateSnippet(), {
 			// eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -104,8 +116,14 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 		this._responseGenerateSnippet = options.responseGenerate;
 		this._responseGenerateFunc = Utils.createAsyncFunction(this.getResponseGenerateSnippet(), {
 			// eslint-disable-next-line @typescript-eslint/no-unused-vars
-			createDefault: () => async ($response: Response, _$factor: InFragment, _$request: PipelineStepData<In>, _$helpers: PipelineStepHelpers, _$: PipelineStepHelpers) => {
-				return await $response.json();
+			createDefault: () => async ($response: Response, _$factor: InFragment, _$request: PipelineStepData<In>, $helpers: PipelineStepHelpers, _$: PipelineStepHelpers) => {
+				const contentEncoding = $response.headers?.get('content-encoding');
+				if (contentEncoding === 'zstd') {
+					const buffer = await $response.buffer();
+					return JSON.parse(await $helpers.$zstd(buffer));
+				} else {
+					return await $response.json();
+				}
 			},
 			getVariableNames: () => this.getResponseGenerateVariableName(),
 			error: (e: Error) => {
@@ -211,6 +229,23 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 		return ['$endpointUrl', '$factor', '$request', ...this.getHelpersVariableNames()];
 	}
 
+	public getTransparentHeaderNames(): Array<string> {
+		return this._transparentHeaderNames ?? [];
+	}
+
+	public getOmittedTransparentHeaderNames(): Array<string> {
+		return this._omittedTransparentHeaderNames ?? [];
+	}
+
+	protected generateTransparentHeaderNames(headerNames?: string, base?: Array<string>): Array<string> {
+		return [
+			...(base ?? []),
+			...new Set(`${headerNames || ''}`.split(';')
+				.map(x => x.trim())
+				.filter(x => x.length !== 0))
+		];
+	}
+
 	public getHeadersGenerateSnippet(): ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>> {
 		return this._headersGenerateSnippet;
 	}
@@ -256,7 +291,36 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 			url = await this._urlGenerateFunc(this.getEndpointUrl(), data, request, $helpers, $helpers);
 			const method = this.getEndpointMethod();
 			const staticHeaders = this.getEndpointHeaders() ?? {};
-			const headers = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
+			const transparentHeaders = (this.getTransparentHeaderNames() ?? []).reduce((headers, name) => {
+				const value = Utils.getValue(data, name);
+				if (value == null) {
+					// no value of given header name, ignored
+				} else if (Array.isArray(value)) {
+					const headerValue = value.filter(v => v != null && `${v}`.length !== 0).join(', ');
+					if (headerValue.length !== 0) {
+						headers[name] = headerValue;
+					}
+				} else if (typeof value === 'object') {
+					Object.keys(value).forEach(key => {
+						const headerValue = value[key];
+						if (headerValue != null) {
+							const s = `${headerValue}`;
+							if (s.length !== 0) {
+								headers[key] = s;
+							}
+						}
+					});
+				} else {
+					const headerValue = `${value}`;
+					if (headerValue.length !== 0) {
+						headers[name] = headerValue;
+					}
+				}
+				return headers;
+			}, {});
+			(this.getOmittedTransparentHeaderNames() ?? []).forEach(name => delete transparentHeaders[name]);
+
+			const generatedHeaders = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
 			// eslint-disable-next-line @typescript-eslint/no-explicit-any
 			let body: any;
 			const bodyUsed = this.isBodyUsed();
@@ -268,8 +332,16 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 			if (body != null && typeof body !== 'string') {
 				body = JSON.stringify(body);
 			}
+			const headers = {...staticHeaders, ...transparentHeaders, ...generatedHeaders};
+			// remove some headers, leave them to fetch to calculate automatically.
+			Object.keys(headers).filter(name => {
+				return ['content-encoding', 'content-length'].includes(name.toLowerCase());
+			}).forEach(name => {
+				delete headers[name];
+			});
+
 			const response = await fetch(url, {
-				method, headers: {...staticHeaders, ...headers}, body,
+				method, headers, body,
 				signal: this.needTimeout() ? (() => {
 					const controller = new AbortController();
 					setTimeout(() => controller.abort(), this.getEndpointTimeout());

package/src/lib/typeorm-step/typeorm-load-many-by-sql-use-cursor-step.ts

@@ -22,6 +22,7 @@ export interface TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In = PipelineS
 	fetchSize?: number;
 	streamTo?: ScriptFuncOrBody<StreamToFunc<In, Item>>;
 	steps?: Array<PipelineStepBuilder>;
+	pauseStreamEnabled?: boolean;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -31,6 +32,7 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 	private readonly _streamToSnippet: ScriptFuncOrBody<StreamToFunc<In, Item>>;
 	private readonly _streamToFunc: StreamToFunc<In, Item>;
 	private readonly _stepBuilders: Array<PipelineStepBuilder>;
+	private readonly _pauseStreamEnabled: boolean;
 
 	public constructor(options: TypeOrmLoadManyBySQLUseCursorPipelineStepOptions<In, Out, InFragment, OutFragment>) {
 		super(options);
@@ -47,6 +49,10 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 			}
 		});
 		this._stepBuilders = options.steps;
+		// for unknown reason, in some environment if the pause and resume functions of readable invoked,
+		// only the first row are returns and end event invoked immediately
+		// so default disable it.
+		this._pauseStreamEnabled = options.pauseStreamEnabled ?? config.getBoolean(`typeorm.${this.getDataSourceName()}.stream.pause.enabled`, false);
 	}
 
 	protected getFetchSize(): number {
@@ -65,6 +71,10 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 		return this._stepBuilders ?? [];
 	}
 
+	protected isPauseStreamEnabled(): boolean {
+		return this._pauseStreamEnabled;
+	}
+
 	protected async doPerform(basis: Undefinable<TypeOrmLoadBasis>, request: PipelineStepData<In>): Promise<Undefinable<OutFragment>> {
 		const {sql, params} = this.getSql(basis, basis?.params);
 		return await this.autoTrans<OutFragment>(async (runner) => {
@@ -136,7 +146,9 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 					reject(e);
 				});
 				readable.on('data', async (data) => {
-					readable.pause();
+					if (this.isPauseStreamEnabled()) {
+						readable.pause();
+					}
 					rows.push(data);
 					await pipe({
 						resolve, reject: async (e: Error) => {
@@ -144,7 +156,9 @@ export class TypeOrmLoadManyBySQLUseCursorPipelineStep<In = PipelineStepPayload,
 							reject(e);
 						}, end: false
 					});
-					readable.resume();
+					if (this.isPauseStreamEnabled()) {
+						readable.resume();
+					}
 				});
 			};
 			return new Promise<OutFragment>((resolve, reject) => read({resolve, reject}));

package/test/step/typeorm-by-cursor.test.ts

@@ -9,7 +9,8 @@ describe('TypeORM Cursor Suite', () => {
 		process.env.CFG_TYPEORM_TEST_HOST = 'localhost';
 		process.env.CFG_TYPEORM_TEST_USERNAME = 'o23';
 		process.env.CFG_TYPEORM_TEST_PASSWORD = 'o23';
-		const type = 'pg' + 'sql';
+		// process.env.CFG_TYPEORM_TEST_STREAM_PAUSE_ENABLED = 'true'
+		const type = 'my' + 'sql';
 		if (type === 'mysql') {
 			process.env.CFG_TYPEORM_TEST_TYPE = 'mysql';
 			process.env.CFG_TYPEORM_TEST_PORT = '3306';

package/test/step/zstd.test.ts

@@ -0,0 +1,65 @@
+import {ZstdInit} from '@oneidentity/zstd-js';
+import {ZSTDDecoder} from 'zstddec';
+
+test('ZStandard compression test', async () => {
+	const data = JSON.stringify({a: new Array(10).fill('1234567890').join('')});
+	const inputArray = new Uint8Array(Buffer.from(data));
+
+	const {ZstdSimple, ZstdStream} = await ZstdInit();
+	// Create some sample data to compress
+	/*
+	 * The required parameter is the data
+	 * It must be a Uint8Array
+	 * */
+	const compressedSimpleData: Uint8Array = ZstdSimple.compress(inputArray, 3);
+	{
+		// console.time('@oneidentity/zstd-js');
+		// for (let i = 0; i < 10000; i++) {
+		// 	const {ZstdSimple} = await ZstdInit();
+		// 	const decompressedArray = ZstdSimple.decompress(compressedSimpleData);
+		// 	new TextDecoder().decode(decompressedArray);
+		// }
+		// console.timeEnd('@oneidentity/zstd-js');
+
+		const {ZstdSimple} = await ZstdInit();
+		const decompressedArray = ZstdSimple.decompress(compressedSimpleData);
+		const decompressedStr = new TextDecoder().decode(decompressedArray);
+		console.log(decompressedStr);
+		console.log(decompressedStr.length);
+		console.log(JSON.parse(decompressedStr));
+	}
+	{
+		console.time('zstddec');
+		for (let i = 0; i < 10; i++) {
+			const decoder = new ZSTDDecoder();
+			await decoder.init();
+			const decompressedArray = decoder.decode(compressedSimpleData);
+			const removeTrailingZeros = (arr: Uint8Array): Uint8Array => {
+				let i = arr.length;
+				while (i > 0 && arr[i - 1] === 0) {
+					i--;
+				}
+				return arr.slice(0, i);
+			};
+			new TextDecoder().decode(removeTrailingZeros(decompressedArray));
+		}
+		console.timeEnd('zstddec');
+
+		const decoder = new ZSTDDecoder();
+		await decoder.init();
+		const decompressedArray = decoder.decode(compressedSimpleData);
+		const removeTrailingZeros = (arr: Uint8Array): Uint8Array => {
+			let i = arr.length;
+			while (i > 0 && arr[i - 1] === 0) {
+				i--;
+			}
+			return arr.slice(0, i);
+		};
+		const decompressedStr = new TextDecoder().decode(removeTrailingZeros(decompressedArray));
+		console.log(decompressedStr);
+		console.log(decompressedStr.length);
+		console.log(JSON.parse(decompressedStr));
+	}
+
+	// const compressedStreamData: Uint8Array = ZstdStream.compress(inputArray);
+}, 30000);