@rainbow-o23/n3 1.0.57 → 1.0.58-alpha.2

package/README.md

@@ -592,13 +592,17 @@ step set. Additionally, nested transactions are also supported, which means Tran
 
 #### Environment Parameters
 
-| Name                                | Type   | Default Value | Comments                                                                                                                          |
-|-------------------------------------|--------|---------------|-----------------------------------------------------------------------------------------------------------------------------------|
-| `endpoints.SYSTEM.ENDPOINT.url`     | string |               | Endpoint URL.                                                                                                                     |
-| `endpoints.SYSTEM.ENDPOINT.headers` | string |               | Endpoint request headers, use global headers if this parameter doesn't present.<br>Format follows `name=value[;name=value[...]]`. |
-| `endpoints.SYSTEM.global.headers`   | string |               | Endpoint system global request headers.<br>Format follows `name=value[;name=value[...]]`.                                         |
-| `endpoints.SYSTEM.ENDPOINT.timeout` | number |               | Endpoint request timeout, in seconds, use global timeout if this parameter doesn't present.                                       |
-| `endpoints.SYSTEM.global.timeout`   | number | -1            | Endpoint system global timeout, in seconds, `-1` represents no timeout.                                                           |
+| Name                                                    | Type   | Default Value | Comments                                                                                                                                                                    |
+|---------------------------------------------------------|--------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `endpoints.SYSTEM.ENDPOINT.url`                         | string |               | Endpoint URL.                                                                                                                                                               |
+| `endpoints.SYSTEM.ENDPOINT.headers`                     | string |               | Endpoint request headers, use global headers if this parameter doesn't present.<br>Format follows `name=value[;name=value[...]]`.                                           |
+| `endpoints.SYSTEM.global.headers`                       | string |               | Endpoint system global request headers.<br>Format follows `name=value[;name=value[...]]`.                                                                                   |
+| `endpoints.SYSTEM.ENDPOINT.headers.transparent`         | string |               | Endpoint request transparent-passed headers names, use global headers if this parameter doesn't present.<br>Format follows `name1[;name2[...]]`.                            |
+| `endpoints.SYSTEM.global.headers.transparent`           | string |               | Endpoint system global request transparent-passed headers names.<br>Format follows `name1[;name2[...]]`.                                                                    |
+| `endpoints.SYSTEM.ENDPOINT.headers.transparent.omitted` | string |               | Endpoint request headers names which omitted from transparent-passed headers, use global headers if this parameter doesn't present.<br>Format follows `name1[;name2[...]]`. |
+| `endpoints.SYSTEM.global.headers.transparent.omitted`   | string |               | Endpoint system global request headers names which omitted from transparent-passed headers.<br>Format follows `name1[;name2[...]]`.                                         |
+| `endpoints.SYSTEM.ENDPOINT.timeout`                     | number |               | Endpoint request timeout, in seconds, use global timeout if this parameter doesn't present.                                                                                 |
+| `endpoints.SYSTEM.global.timeout`                       | number | -1            | Endpoint system global timeout, in seconds, `-1` represents no timeout.                                                                                                     |
 
 `SYSTEM` represents endpoint system, `ENDPOINT` represents endpoint url. For example:
 
@@ -611,18 +615,60 @@ CFG_ENDPOINTS_ORDER_PAYMENT_URL=https://order.com/payment
 
 #### Constructor Parameters
 
-| Name                 | Type                                                                                                   | Default Value | Comments                                                                                                             |
-|----------------------|--------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------|
-| endpointSystemCode   | string                                                                                                 |               | Endpoint system code.                                                                                                |
-| endpointName         | string                                                                                                 |               | Endpoint name.                                                                                                       |
-| urlGenerate          | ScriptFuncOrBody\<HttpGenerateUrl>                                                                     |               | Endpoint url generator, `$endpointUrl`.                                                                              |
-| method               | string                                                                                                 |               | Http method, default `post`.                                                                                         |
-| timeout              | number                                                                                                 |               | Endpoint timeout, in seconds.                                                                                        |
-| headersGenerate      | ScriptFuncOrBody\<HttpGenerateHeaders>                                                                 |               | Endpoint request headers generator.                                                                                  |
-| bodyUsed             | boolean                                                                                                |               | Send request with body or not, or automatically disregards the body when sending a `get` request when not specified. |
-| bodyGenerate         | ScriptFuncOrBody\<HttpGenerateBody>                                                                    |               | Endpoint request body generator.                                                                                     |
-| responseGenerate     | ScriptFuncOrBody\<HttpGenerateResponse>                                                                |               | Endpoint response body generator, `$response`.                                                                       |
-| responseErrorHandles | ScriptFuncOrBody\<HttpHandleError><br>or<br>{[key: HttpErrorCode]: ScriptFuncOrBody\<HttpHandleError>} |               | Endpoint response error handlers.                                                                                    |
+| Name                          | Type                                                                                                   | Default Value | Comments                                                                                                             |
+|-------------------------------|--------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------|
+| endpointSystemCode            | string                                                                                                 |               | Endpoint system code.                                                                                                |
+| endpointName                  | string                                                                                                 |               | Endpoint name.                                                                                                       |
+| urlGenerate                   | ScriptFuncOrBody\<HttpGenerateUrl>                                                                     |               | Endpoint url generator, `$endpointUrl`.                                                                              |
+| method                        | string                                                                                                 |               | Http method, default `post`.                                                                                         |
+| timeout                       | number                                                                                                 |               | Endpoint timeout, in seconds.                                                                                        |
+| transparentHeaderNames        | Array<string>                                                                                          |               | Transparent-passed request headers names.                                                                            |
+| omittedTransparentHeaderNames | Array<string>                                                                                          |               | Omitted names of transparently-passed request headers.                                                               |
+| headersGenerate               | ScriptFuncOrBody\<HttpGenerateHeaders>                                                                 |               | Endpoint request headers generator.                                                                                  |
+| bodyUsed                      | boolean                                                                                                |               | Send request with body or not, or automatically disregards the body when sending a `get` request when not specified. |
+| bodyGenerate                  | ScriptFuncOrBody\<HttpGenerateBody>                                                                    |               | Endpoint request body generator.                                                                                     |
+| responseGenerate              | ScriptFuncOrBody\<HttpGenerateResponse>                                                                |               | Endpoint response body generator, `$response`.                                                                       |
+| responseErrorHandles          | ScriptFuncOrBody\<HttpHandleError><br>or<br>{[key: HttpErrorCode]: ScriptFuncOrBody\<HttpHandleError>} |               | Endpoint response error handlers.                                                                                    |
+
+- `transparentHeaderNames` and `omittedTransparentHeaderNames`:
+  Use `transparentHeaderNames` to specify the names of request headers whose values need to be transparently passed from the input
+  parameters to the downstream service. Separate the names with ';'. The names support using `.` for connection so that values from
+  multi-level objects can be directly retrieved. For example, `account.name` will retrieve the value of the `name` property from the
+  `account` property of the input object. When writing the values into the header values, the following rules apply:
+	- If the value is an array, use `, ` to connect the elements. `null` and empty strings will be filtered out.
+	- If the value is an object, use the object's keys to generate multiple headers. `null` and empty strings will be filtered out.
+	- For other values, convert them to strings. `null` and empty strings will be filtered out.
+	- Note that an empty string does not include blank strings, and no automatic trimming will be performed.
+
+  If the `transparentHeaderNames` at the step level is not defined, use the definition in
+  `endpoints.SYSTEM.ENDPOINT.headers.transparent`. If it is also not defined at the endpoint level, use the definition in
+  `endpoints.SYSTEM.global.headers.transparent`.
+
+  After obtaining the transparently passed request headers, check the definition of `omittedTransparentHeaderNames`. If it is defined,
+  remove the corresponding headers from the headers. `omittedTransparentHeaderNames` is case-insensitive. Similarly, if the
+  `omittedTransparentHeaderNames` at the step level is not defined, use the definition in
+  `endpoints.SYSTEM.ENDPOINT.headers.transparent.omitted`. If it is also not defined at the endpoint level, use the definition in
+  `endpoints.SYSTEM.global.headers.transparent.omitted`.
+
+  For example:
+  If the input data contains `{account: {name: 'John', token: '******'}}` and `transparentHeaderNames` is defined as `account`, then two
+  transparently passed headers will be obtained: `name=John` and `token=******`. At this time, if `omittedTransparentHeaderNames` is defined
+  as `name`, the headers that will be finally transparently passed to the downstream service are `token=******`, and `name` will be ignored.
+
+## Request headers
+
+There are three ways to transmit request headers to downstream services. In order of priority from high to low, they are `headersGenerate`,
+`headers.transparent`, and `headers`. If a header appears in a higher-priority method, the header with the same name generated by a
+lower-priority method will be ignored. Note that the matching is case-sensitive.
+
+Normally, if you need to transparently pass the request headers from the client to the downstream service, you should use `headers: true` in
+the pipeline definition. Then you can directly use `transparentHeaderNames: headers` to obtain all the request headers, and then use
+`omittedTransparentHeaderNames` for necessary filtering.
+
+> It should be noted that since the fetch step initiates a new request to the downstream service, its request structure and data will be
+> modified or reset according to requirements. Therefore, even if you need to transparently pass the request headers, some of the headers
+> are still not applicable. So by default, the two headers `content-encoding` and `content-length` will be filtered out. No matter how the
+> request headers are generated in the above process, these two headers are always automatically generated by `node-fetch`.
 
 ## Installation
 

package/index.cjs

@@ -1281,6 +1281,8 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     _endpointTimeout;
     _urlGenerateSnippet;
     _urlGenerateFunc;
+    _transparentHeaderNames;
+    _omittedTransparentHeaderNames;
     _headersGenerateSnippet;
     _headersGenerateFunc;
     _bodyUsed;
@@ -1311,6 +1313,10 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
                 throw e;
             }
         });
+        this._transparentHeaderNames = options.transparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent`)));
+        this._omittedTransparentHeaderNames = options.omittedTransparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent.omitted`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent.omitted`)));
         this._headersGenerateSnippet = options.headersGenerate;
         this._headersGenerateFunc = Utils.createAsyncFunction(this.getHeadersGenerateSnippet(), {
             createDefault: () => async (_$factor, _$request, _$helpers, _$) => (void 0),
@@ -1334,8 +1340,15 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
         });
         this._responseGenerateSnippet = options.responseGenerate;
         this._responseGenerateFunc = Utils.createAsyncFunction(this.getResponseGenerateSnippet(), {
-            createDefault: () => async ($response, _$factor, _$request, _$helpers, _$) => {
-                return await $response.json();
+            createDefault: () => async ($response, _$factor, _$request, $helpers, _$) => {
+                const contentEncoding = $response.headers?.get('content-encoding');
+                if (contentEncoding === 'zstd') {
+                    const buffer = await $response.buffer();
+                    return JSON.parse(await $helpers.$zstd(buffer));
+                }
+                else {
+                    return await $response.json();
+                }
             },
             getVariableNames: () => this.getResponseGenerateVariableName(),
             error: (e) => {
@@ -1427,6 +1440,20 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     getUrlGenerateVariableName() {
         return ['$endpointUrl', '$factor', '$request', ...this.getHelpersVariableNames()];
     }
+    getTransparentHeaderNames() {
+        return this._transparentHeaderNames ?? [];
+    }
+    getOmittedTransparentHeaderNames() {
+        return this._omittedTransparentHeaderNames ?? [];
+    }
+    generateTransparentHeaderNames(headerNames, base) {
+        return [
+            ...(base ?? []),
+            ...new Set(`${headerNames || ''}`.split(';')
+                .map(x => x.trim())
+                .filter(x => x.length !== 0))
+        ];
+    }
     getHeadersGenerateSnippet() {
         return this._headersGenerateSnippet;
     }
@@ -1458,7 +1485,37 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             url = await this._urlGenerateFunc(this.getEndpointUrl(), data, request, $helpers, $helpers);
             const method = this.getEndpointMethod();
             const staticHeaders = this.getEndpointHeaders() ?? {};
-            const headers = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
+            const transparentHeaders = (this.getTransparentHeaderNames() ?? []).reduce((headers, name) => {
+                const value = Utils.getValue(data, name);
+                if (value == null) {
+                }
+                else if (Array.isArray(value)) {
+                    const headerValue = value.filter(v => v != null && `${v}`.length !== 0).join(', ');
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                else if (typeof value === 'object') {
+                    Object.keys(value).forEach(key => {
+                        const headerValue = value[key];
+                        if (headerValue != null) {
+                            const s = `${headerValue}`;
+                            if (s.length !== 0) {
+                                headers[key] = s;
+                            }
+                        }
+                    });
+                }
+                else {
+                    const headerValue = `${value}`;
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                return headers;
+            }, {});
+            (this.getOmittedTransparentHeaderNames() ?? []).forEach(name => delete transparentHeaders[name]);
+            const generatedHeaders = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
             let body;
             const bodyUsed = this.isBodyUsed();
             if (bodyUsed === true || (bodyUsed == null && method !== 'get')) {
@@ -1470,8 +1527,14 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             if (body != null && typeof body !== 'string') {
                 body = JSON.stringify(body);
             }
+            const headers = { ...staticHeaders, ...transparentHeaders, ...generatedHeaders };
+            Object.keys(headers).filter(name => {
+                return ['content-encoding', 'content-length'].includes(name.toLowerCase());
+            }).forEach(name => {
+                delete headers[name];
+            });
             const response = await fetch(url, {
-                method, headers: { ...staticHeaders, ...headers }, body,
+                method, headers, body,
                 signal: this.needTimeout() ? (() => {
                     const controller = new AbortController();
                     setTimeout(() => controller.abort(), this.getEndpointTimeout());

package/index.js

@@ -1279,6 +1279,8 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     _endpointTimeout;
     _urlGenerateSnippet;
     _urlGenerateFunc;
+    _transparentHeaderNames;
+    _omittedTransparentHeaderNames;
     _headersGenerateSnippet;
     _headersGenerateFunc;
     _bodyUsed;
@@ -1309,6 +1311,10 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
                 throw e;
             }
         });
+        this._transparentHeaderNames = options.transparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent`)));
+        this._omittedTransparentHeaderNames = options.omittedTransparentHeaderNames
+            ?? this.generateTransparentHeaderNames(config.getString(`endpoints.${endpointKey}.headers.transparent.omitted`), this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent.omitted`)));
         this._headersGenerateSnippet = options.headersGenerate;
         this._headersGenerateFunc = Utils.createAsyncFunction(this.getHeadersGenerateSnippet(), {
             createDefault: () => async (_$factor, _$request, _$helpers, _$) => (void 0),
@@ -1332,8 +1338,15 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
         });
         this._responseGenerateSnippet = options.responseGenerate;
         this._responseGenerateFunc = Utils.createAsyncFunction(this.getResponseGenerateSnippet(), {
-            createDefault: () => async ($response, _$factor, _$request, _$helpers, _$) => {
-                return await $response.json();
+            createDefault: () => async ($response, _$factor, _$request, $helpers, _$) => {
+                const contentEncoding = $response.headers?.get('content-encoding');
+                if (contentEncoding === 'zstd') {
+                    const buffer = await $response.buffer();
+                    return JSON.parse(await $helpers.$zstd(buffer));
+                }
+                else {
+                    return await $response.json();
+                }
             },
             getVariableNames: () => this.getResponseGenerateVariableName(),
             error: (e) => {
@@ -1425,6 +1438,20 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
     getUrlGenerateVariableName() {
         return ['$endpointUrl', '$factor', '$request', ...this.getHelpersVariableNames()];
     }
+    getTransparentHeaderNames() {
+        return this._transparentHeaderNames ?? [];
+    }
+    getOmittedTransparentHeaderNames() {
+        return this._omittedTransparentHeaderNames ?? [];
+    }
+    generateTransparentHeaderNames(headerNames, base) {
+        return [
+            ...(base ?? []),
+            ...new Set(`${headerNames || ''}`.split(';')
+                .map(x => x.trim())
+                .filter(x => x.length !== 0))
+        ];
+    }
     getHeadersGenerateSnippet() {
         return this._headersGenerateSnippet;
     }
@@ -1456,7 +1483,37 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             url = await this._urlGenerateFunc(this.getEndpointUrl(), data, request, $helpers, $helpers);
             const method = this.getEndpointMethod();
             const staticHeaders = this.getEndpointHeaders() ?? {};
-            const headers = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
+            const transparentHeaders = (this.getTransparentHeaderNames() ?? []).reduce((headers, name) => {
+                const value = Utils.getValue(data, name);
+                if (value == null) {
+                }
+                else if (Array.isArray(value)) {
+                    const headerValue = value.filter(v => v != null && `${v}`.length !== 0).join(', ');
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                else if (typeof value === 'object') {
+                    Object.keys(value).forEach(key => {
+                        const headerValue = value[key];
+                        if (headerValue != null) {
+                            const s = `${headerValue}`;
+                            if (s.length !== 0) {
+                                headers[key] = s;
+                            }
+                        }
+                    });
+                }
+                else {
+                    const headerValue = `${value}`;
+                    if (headerValue.length !== 0) {
+                        headers[name] = headerValue;
+                    }
+                }
+                return headers;
+            }, {});
+            (this.getOmittedTransparentHeaderNames() ?? []).forEach(name => delete transparentHeaders[name]);
+            const generatedHeaders = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
             let body;
             const bodyUsed = this.isBodyUsed();
             if (bodyUsed === true || (bodyUsed == null && method !== 'get')) {
@@ -1468,8 +1525,14 @@ class FetchPipelineStep extends AbstractFragmentaryPipelineStep {
             if (body != null && typeof body !== 'string') {
                 body = JSON.stringify(body);
             }
+            const headers = { ...staticHeaders, ...transparentHeaders, ...generatedHeaders };
+            Object.keys(headers).filter(name => {
+                return ['content-encoding', 'content-length'].includes(name.toLowerCase());
+            }).forEach(name => {
+                delete headers[name];
+            });
             const response = await fetch(url, {
-                method, headers: { ...staticHeaders, ...headers }, body,
+                method, headers, body,
                 signal: this.needTimeout() ? (() => {
                     const controller = new AbortController();
                     setTimeout(() => controller.abort(), this.getEndpointTimeout());

package/lib/http/fetch-step.d.ts

@@ -7,6 +7,8 @@ export interface FetchPipelineStepOptions<In = PipelineStepPayload, Out = Pipeli
     urlGenerate?: ScriptFuncOrBody<HttpGenerateUrl<In, InFragment>>;
     method?: string;
     timeout?: number;
+    transparentHeaderNames?: Array<string>;
+    omittedTransparentHeaderNames?: Array<string>;
     headersGenerate?: ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
     bodyUsed?: boolean;
     bodyGenerate?: ScriptFuncOrBody<HttpGenerateBody<In, InFragment, BodyData>>;
@@ -24,6 +26,8 @@ export declare class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineS
     private readonly _endpointTimeout;
     private readonly _urlGenerateSnippet;
     private readonly _urlGenerateFunc;
+    private readonly _transparentHeaderNames;
+    private readonly _omittedTransparentHeaderNames;
     private readonly _headersGenerateSnippet;
     private readonly _headersGenerateFunc;
     private readonly _bodyUsed;
@@ -44,6 +48,9 @@ export declare class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineS
     needTimeout(): boolean;
     getUrlGenerateSnippet(): ScriptFuncOrBody<HttpGenerateUrl<In, InFragment>>;
     protected getUrlGenerateVariableName(): Array<string>;
+    getTransparentHeaderNames(): Array<string>;
+    getOmittedTransparentHeaderNames(): Array<string>;
+    protected generateTransparentHeaderNames(headerNames?: string, base?: Array<string>): Array<string>;
     getHeadersGenerateSnippet(): ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
     protected getHeadersGenerateVariableNames(): Array<string>;
     protected isBodyUsed(): boolean | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rainbow-o23/n3",
-	"version": "1.0.57",
+	"version": "1.0.58-alpha.2",
 	"description": "o23 pipelines",
 	"main": "index.cjs",
 	"module": "index.js",
@@ -21,7 +21,7 @@
 		"url": "https://github.com/InsureMO/rainbow-o23/issues"
 	},
 	"dependencies": {
-		"@rainbow-o23/n1": "1.0.57",
+		"@rainbow-o23/n1": "1.0.58-alpha.2",
 		"node-fetch": "2.6.7",
 		"typeorm": "^0.3.20",
 		"typescript": "5.5.4"
@@ -30,6 +30,7 @@
 		"@babel/core": "^7.23.9",
 		"@babel/preset-env": "^7.23.9",
 		"@babel/preset-typescript": "^7.23.3",
+		"@oneidentity/zstd-js": "^1.0.3",
 		"@rollup/plugin-babel": "^6.0.4",
 		"@rollup/plugin-eslint": "^9.0.3",
 		"@types/better-sqlite3": "^7.6.6",

package/src/lib/http/fetch-step.ts

@@ -23,6 +23,8 @@ export interface FetchPipelineStepOptions<In = PipelineStepPayload, Out = Pipeli
 	method?: string;
 	/** on seconds */
 	timeout?: number;
+	transparentHeaderNames?: Array<string>;
+	omittedTransparentHeaderNames?: Array<string>;
 	headersGenerate?: ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
 	bodyUsed?: boolean;
 	bodyGenerate?: ScriptFuncOrBody<HttpGenerateBody<In, InFragment, BodyData>>;
@@ -42,6 +44,8 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 	private readonly _endpointTimeout: number;
 	private readonly _urlGenerateSnippet: ScriptFuncOrBody<HttpGenerateUrl<In, InFragment>>;
 	private readonly _urlGenerateFunc: HttpGenerateUrl<In, InFragment>;
+	private readonly _transparentHeaderNames: Array<string>;
+	private readonly _omittedTransparentHeaderNames: Array<string>;
 	private readonly _headersGenerateSnippet: ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>>;
 	private readonly _headersGenerateFunc: HttpGenerateHeaders<In, InFragment>;
 	private readonly _bodyUsed: boolean;
@@ -78,6 +82,14 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 				throw e;
 			}
 		});
+		this._transparentHeaderNames = options.transparentHeaderNames
+			?? this.generateTransparentHeaderNames(
+				config.getString(`endpoints.${endpointKey}.headers.transparent`),
+				this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent`)));
+		this._omittedTransparentHeaderNames = options.omittedTransparentHeaderNames
+			?? this.generateTransparentHeaderNames(
+				config.getString(`endpoints.${endpointKey}.headers.transparent.omitted`),
+				this.generateTransparentHeaderNames(config.getString(`endpoints.${this.getEndpointSystemCode()}.global.headers.transparent.omitted`)));
 		this._headersGenerateSnippet = options.headersGenerate;
 		this._headersGenerateFunc = Utils.createAsyncFunction(this.getHeadersGenerateSnippet(), {
 			// eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -104,8 +116,14 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 		this._responseGenerateSnippet = options.responseGenerate;
 		this._responseGenerateFunc = Utils.createAsyncFunction(this.getResponseGenerateSnippet(), {
 			// eslint-disable-next-line @typescript-eslint/no-unused-vars
-			createDefault: () => async ($response: Response, _$factor: InFragment, _$request: PipelineStepData<In>, _$helpers: PipelineStepHelpers, _$: PipelineStepHelpers) => {
-				return await $response.json();
+			createDefault: () => async ($response: Response, _$factor: InFragment, _$request: PipelineStepData<In>, $helpers: PipelineStepHelpers, _$: PipelineStepHelpers) => {
+				const contentEncoding = $response.headers?.get('content-encoding');
+				if (contentEncoding === 'zstd') {
+					const buffer = await $response.buffer();
+					return JSON.parse(await $helpers.$zstd(buffer));
+				} else {
+					return await $response.json();
+				}
 			},
 			getVariableNames: () => this.getResponseGenerateVariableName(),
 			error: (e: Error) => {
@@ -211,6 +229,23 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 		return ['$endpointUrl', '$factor', '$request', ...this.getHelpersVariableNames()];
 	}
 
+	public getTransparentHeaderNames(): Array<string> {
+		return this._transparentHeaderNames ?? [];
+	}
+
+	public getOmittedTransparentHeaderNames(): Array<string> {
+		return this._omittedTransparentHeaderNames ?? [];
+	}
+
+	protected generateTransparentHeaderNames(headerNames?: string, base?: Array<string>): Array<string> {
+		return [
+			...(base ?? []),
+			...new Set(`${headerNames || ''}`.split(';')
+				.map(x => x.trim())
+				.filter(x => x.length !== 0))
+		];
+	}
+
 	public getHeadersGenerateSnippet(): ScriptFuncOrBody<HttpGenerateHeaders<In, InFragment>> {
 		return this._headersGenerateSnippet;
 	}
@@ -256,7 +291,36 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 			url = await this._urlGenerateFunc(this.getEndpointUrl(), data, request, $helpers, $helpers);
 			const method = this.getEndpointMethod();
 			const staticHeaders = this.getEndpointHeaders() ?? {};
-			const headers = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
+			const transparentHeaders = (this.getTransparentHeaderNames() ?? []).reduce((headers, name) => {
+				const value = Utils.getValue(data, name);
+				if (value == null) {
+					// no value of given header name, ignored
+				} else if (Array.isArray(value)) {
+					const headerValue = value.filter(v => v != null && `${v}`.length !== 0).join(', ');
+					if (headerValue.length !== 0) {
+						headers[name] = headerValue;
+					}
+				} else if (typeof value === 'object') {
+					Object.keys(value).forEach(key => {
+						const headerValue = value[key];
+						if (headerValue != null) {
+							const s = `${headerValue}`;
+							if (s.length !== 0) {
+								headers[key] = s;
+							}
+						}
+					});
+				} else {
+					const headerValue = `${value}`;
+					if (headerValue.length !== 0) {
+						headers[name] = headerValue;
+					}
+				}
+				return headers;
+			}, {});
+			(this.getOmittedTransparentHeaderNames() ?? []).forEach(name => delete transparentHeaders[name]);
+
+			const generatedHeaders = await this._headersGenerateFunc(data, request, $helpers, $helpers) ?? {};
 			// eslint-disable-next-line @typescript-eslint/no-explicit-any
 			let body: any;
 			const bodyUsed = this.isBodyUsed();
@@ -268,8 +332,16 @@ export class FetchPipelineStep<In = PipelineStepPayload, Out = PipelineStepPaylo
 			if (body != null && typeof body !== 'string') {
 				body = JSON.stringify(body);
 			}
+			const headers = {...staticHeaders, ...transparentHeaders, ...generatedHeaders};
+			// remove some headers, leave them to fetch to calculate automatically.
+			Object.keys(headers).filter(name => {
+				return ['content-encoding', 'content-length'].includes(name.toLowerCase());
+			}).forEach(name => {
+				delete headers[name];
+			});
+
 			const response = await fetch(url, {
-				method, headers: {...staticHeaders, ...headers}, body,
+				method, headers, body,
 				signal: this.needTimeout() ? (() => {
 					const controller = new AbortController();
 					setTimeout(() => controller.abort(), this.getEndpointTimeout());

package/test/step/zstd.test.ts

@@ -0,0 +1,65 @@
+import {ZstdInit} from '@oneidentity/zstd-js';
+import {ZSTDDecoder} from 'zstddec';
+
+test('ZStandard compression test', async () => {
+	const data = JSON.stringify({a: new Array(10).fill('1234567890').join('')});
+	const inputArray = new Uint8Array(Buffer.from(data));
+
+	const {ZstdSimple, ZstdStream} = await ZstdInit();
+	// Create some sample data to compress
+	/*
+	 * The required parameter is the data
+	 * It must be a Uint8Array
+	 * */
+	const compressedSimpleData: Uint8Array = ZstdSimple.compress(inputArray, 3);
+	{
+		// console.time('@oneidentity/zstd-js');
+		// for (let i = 0; i < 10000; i++) {
+		// 	const {ZstdSimple} = await ZstdInit();
+		// 	const decompressedArray = ZstdSimple.decompress(compressedSimpleData);
+		// 	new TextDecoder().decode(decompressedArray);
+		// }
+		// console.timeEnd('@oneidentity/zstd-js');
+
+		const {ZstdSimple} = await ZstdInit();
+		const decompressedArray = ZstdSimple.decompress(compressedSimpleData);
+		const decompressedStr = new TextDecoder().decode(decompressedArray);
+		console.log(decompressedStr);
+		console.log(decompressedStr.length);
+		console.log(JSON.parse(decompressedStr));
+	}
+	{
+		console.time('zstddec');
+		for (let i = 0; i < 10; i++) {
+			const decoder = new ZSTDDecoder();
+			await decoder.init();
+			const decompressedArray = decoder.decode(compressedSimpleData);
+			const removeTrailingZeros = (arr: Uint8Array): Uint8Array => {
+				let i = arr.length;
+				while (i > 0 && arr[i - 1] === 0) {
+					i--;
+				}
+				return arr.slice(0, i);
+			};
+			new TextDecoder().decode(removeTrailingZeros(decompressedArray));
+		}
+		console.timeEnd('zstddec');
+
+		const decoder = new ZSTDDecoder();
+		await decoder.init();
+		const decompressedArray = decoder.decode(compressedSimpleData);
+		const removeTrailingZeros = (arr: Uint8Array): Uint8Array => {
+			let i = arr.length;
+			while (i > 0 && arr[i - 1] === 0) {
+				i--;
+			}
+			return arr.slice(0, i);
+		};
+		const decompressedStr = new TextDecoder().decode(removeTrailingZeros(decompressedArray));
+		console.log(decompressedStr);
+		console.log(decompressedStr.length);
+		console.log(JSON.parse(decompressedStr));
+	}
+
+	// const compressedStreamData: Uint8Array = ZstdStream.compress(inputArray);
+}, 30000);