@transai/connector-runner-api 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.json +18 -0
- package/CHANGELOG.md +19 -0
- package/README.md +11 -0
- package/jest.config.ts +10 -0
- package/package.json +16 -0
- package/project.json +29 -0
- package/src/index.ts +1 -0
- package/src/lib/connector-runner-api.ts +53 -0
- package/src/lib/extractor.service.spec.ts +90 -0
- package/src/lib/extractor.service.ts +97 -0
- package/src/lib/http-client-authentication.ts +108 -0
- package/src/lib/result.handler.spec.ts +135 -0
- package/src/lib/result.handler.ts +100 -0
- package/src/lib/types.ts +38 -0
- package/tsconfig.json +22 -0
- package/tsconfig.lib.json +10 -0
- package/tsconfig.spec.json +14 -0
package/.eslintrc.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": ["../../.eslintrc.json"],
|
|
3
|
+
"ignorePatterns": ["!**/*"],
|
|
4
|
+
"overrides": [
|
|
5
|
+
{
|
|
6
|
+
"files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
|
|
7
|
+
"rules": {}
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"files": ["*.ts", "*.tsx"],
|
|
11
|
+
"rules": {}
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"files": ["*.js", "*.jsx"],
|
|
15
|
+
"rules": {}
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
}
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
## 0.1.0 (2025-11-19)
|
|
2
|
+
|
|
3
|
+
### 🚀 Features
|
|
4
|
+
|
|
5
|
+
- fix connector packages publish ([517d2fd2](https://github.com/xip-online-applications/xod-core/commit/517d2fd2))
|
|
6
|
+
- **XODO-1134:** added generic API connector ([#883](https://github.com/xip-online-applications/xod-core/pull/883))
|
|
7
|
+
- **XODO-1029 XODO-620:** added initial bystronic connector runner & opcua client ([#776](https://github.com/xip-online-applications/xod-core/pull/776))
|
|
8
|
+
- **XODO-727:** connector orchestration config through API call ([#797](https://github.com/xip-online-applications/xod-core/pull/797))
|
|
9
|
+
- **XODO-852:** made connectors publish to npm ([89cd40eb](https://github.com/xip-online-applications/xod-core/commit/89cd40eb))
|
|
10
|
+
- **XODO-852:** made connectors buildable ([6ca5f111](https://github.com/xip-online-applications/xod-core/commit/6ca5f111))
|
|
11
|
+
|
|
12
|
+
### 🧱 Updated Dependencies
|
|
13
|
+
|
|
14
|
+
- Updated @transai/connector-runtime-sdk to 0.1.0
|
|
15
|
+
|
|
16
|
+
### ❤️ Thank You
|
|
17
|
+
|
|
18
|
+
- Rene Heijdens @H31nz3l
|
|
19
|
+
- Youri Lefers @yourilefers
|
package/README.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# connector-runner-api
|
|
2
|
+
|
|
3
|
+
This library was generated with [Nx](https://nx.dev).
|
|
4
|
+
|
|
5
|
+
## Building
|
|
6
|
+
|
|
7
|
+
Run `nx build connector-runner-api` to build the library.
|
|
8
|
+
|
|
9
|
+
## Running unit tests
|
|
10
|
+
|
|
11
|
+
Run `nx test connector-runner-api` to execute the unit tests via [Jest](https://jestjs.io).
|
package/jest.config.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export default {
|
|
2
|
+
displayName: 'connector-runner-api',
|
|
3
|
+
preset: '../../jest.preset.js',
|
|
4
|
+
testEnvironment: 'node',
|
|
5
|
+
transform: {
|
|
6
|
+
'^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
|
|
7
|
+
},
|
|
8
|
+
moduleFileExtensions: ['ts', 'js', 'html'],
|
|
9
|
+
coverageDirectory: '../../coverage/libs/connector-runner-api',
|
|
10
|
+
};
|
package/package.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@transai/connector-runner-api",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"publishConfig": {
|
|
5
|
+
"access": "public"
|
|
6
|
+
},
|
|
7
|
+
"license": "LGPL-3.0-or-later",
|
|
8
|
+
"author": {
|
|
9
|
+
"name": "transAI",
|
|
10
|
+
"email": "samen@transai.com",
|
|
11
|
+
"url": "https://transai.com"
|
|
12
|
+
},
|
|
13
|
+
"type": "commonjs",
|
|
14
|
+
"main": "./index.js",
|
|
15
|
+
"typings": "./index.d.ts"
|
|
16
|
+
}
|
package/project.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "connector-runner-api",
|
|
3
|
+
"$schema": "../../node_modules/nx/schemas/project-schema.json",
|
|
4
|
+
"sourceRoot": "libs/connector-runner-api/src",
|
|
5
|
+
"projectType": "library",
|
|
6
|
+
"tags": [
|
|
7
|
+
"connector:runner",
|
|
8
|
+
"connector:runner-sdk",
|
|
9
|
+
"connector:api",
|
|
10
|
+
"connector:source",
|
|
11
|
+
"provider:@transai"
|
|
12
|
+
],
|
|
13
|
+
"targets": {
|
|
14
|
+
"build": {
|
|
15
|
+
"executor": "@transai/tools:connector",
|
|
16
|
+
"outputs": ["{options.outputPath}"]
|
|
17
|
+
},
|
|
18
|
+
"lint": {
|
|
19
|
+
"executor": "@nx/eslint:lint"
|
|
20
|
+
},
|
|
21
|
+
"test": {
|
|
22
|
+
"executor": "@nx/jest:jest",
|
|
23
|
+
"outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
|
|
24
|
+
"options": {
|
|
25
|
+
"jestConfig": "libs/connector-runner-api/jest.config.ts"
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './lib/connector-runner-api';
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ConnectorRuntimeSDK,
|
|
3
|
+
ConnectorSDKInterface,
|
|
4
|
+
HttpClientSDKInterface,
|
|
5
|
+
} from '@transai/connector-runtime-sdk';
|
|
6
|
+
import { ConnectorInterface } from '@xip-online-data/types';
|
|
7
|
+
|
|
8
|
+
import { ExtractorService } from './extractor.service';
|
|
9
|
+
import { HttpClientAuthentication } from './http-client-authentication';
|
|
10
|
+
import { ResultHandler } from './result.handler';
|
|
11
|
+
import { ConnectorConfig } from './types';
|
|
12
|
+
|
|
13
|
+
export class ConnectorRunnerApi extends ConnectorRuntimeSDK<ConnectorConfig> {
|
|
14
|
+
readonly #httpClient?: HttpClientSDKInterface = undefined;
|
|
15
|
+
|
|
16
|
+
constructor(
|
|
17
|
+
connector: ConnectorInterface,
|
|
18
|
+
connectorSDK: ConnectorSDKInterface,
|
|
19
|
+
) {
|
|
20
|
+
super(connector, connectorSDK);
|
|
21
|
+
|
|
22
|
+
const { config } = this.connectorSDK;
|
|
23
|
+
if (config.url) {
|
|
24
|
+
this.#httpClient = this.connectorSDK
|
|
25
|
+
.httpClient({
|
|
26
|
+
baseUrl: config.url,
|
|
27
|
+
})
|
|
28
|
+
.setRequestOptionsFormatter(
|
|
29
|
+
HttpClientAuthentication.createForAuthConfig(
|
|
30
|
+
config,
|
|
31
|
+
this.connectorSDK.httpClient(),
|
|
32
|
+
),
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
override init = async (): Promise<void> => {
|
|
38
|
+
const { config } = this.connectorSDK;
|
|
39
|
+
const resultHandler = new ResultHandler(this.connectorSDK);
|
|
40
|
+
|
|
41
|
+
(config.apiCalls ?? []).forEach((apiConfig) => {
|
|
42
|
+
this.connectorSDK.processing.registerInterval(
|
|
43
|
+
apiConfig.interval,
|
|
44
|
+
new ExtractorService(
|
|
45
|
+
this.connectorSDK,
|
|
46
|
+
apiConfig,
|
|
47
|
+
resultHandler,
|
|
48
|
+
this.#httpClient,
|
|
49
|
+
),
|
|
50
|
+
);
|
|
51
|
+
});
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CompileDelegate,
|
|
3
|
+
ConnectorSDKInterface,
|
|
4
|
+
HttpClientSDKInterface,
|
|
5
|
+
} from '@transai/connector-runtime-sdk';
|
|
6
|
+
|
|
7
|
+
import { ExtractorService } from './extractor.service';
|
|
8
|
+
import { ResultHandler } from './result.handler';
|
|
9
|
+
import { ConnectorConfig } from './types';
|
|
10
|
+
|
|
11
|
+
describe('ExtractorService', () => {
|
|
12
|
+
let service: ExtractorService;
|
|
13
|
+
|
|
14
|
+
const callConfig = {
|
|
15
|
+
name: 'TestCall',
|
|
16
|
+
interval: 60,
|
|
17
|
+
offsetFilePrefix: 'test_offset',
|
|
18
|
+
url: 'https://localhost:443',
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const httpClientMock = {
|
|
22
|
+
request: jest.fn().mockResolvedValue({
|
|
23
|
+
status: 200,
|
|
24
|
+
data: 'some-result-data',
|
|
25
|
+
}),
|
|
26
|
+
setRequestOptionsFormatter: jest.fn().mockReturnThis(),
|
|
27
|
+
} as unknown as HttpClientSDKInterface;
|
|
28
|
+
let sdkMock: ConnectorSDKInterface<ConnectorConfig>;
|
|
29
|
+
let resultHandlerMock: ResultHandler;
|
|
30
|
+
|
|
31
|
+
beforeEach(() => {
|
|
32
|
+
sdkMock = {
|
|
33
|
+
logger: {
|
|
34
|
+
info: jest.fn(),
|
|
35
|
+
debug: jest.fn(),
|
|
36
|
+
error: jest.fn(),
|
|
37
|
+
verbose: jest.fn(),
|
|
38
|
+
warn: jest.fn(),
|
|
39
|
+
},
|
|
40
|
+
offsetStore: {
|
|
41
|
+
getOffset: jest.fn().mockResolvedValue({
|
|
42
|
+
id: 0,
|
|
43
|
+
timestamp: new Date().getTime(),
|
|
44
|
+
rawTimestamp: new Date().toISOString(),
|
|
45
|
+
isoDate: new Date().toISOString(),
|
|
46
|
+
}),
|
|
47
|
+
},
|
|
48
|
+
templating: {
|
|
49
|
+
compile: jest
|
|
50
|
+
.fn()
|
|
51
|
+
.mockImplementation(
|
|
52
|
+
(input) =>
|
|
53
|
+
jest.fn().mockReturnValue(`compiled-${input}`) as CompileDelegate,
|
|
54
|
+
),
|
|
55
|
+
},
|
|
56
|
+
config: {},
|
|
57
|
+
httpClient: jest.fn().mockReturnValue(httpClientMock),
|
|
58
|
+
} as unknown as ConnectorSDKInterface<ConnectorConfig>;
|
|
59
|
+
|
|
60
|
+
resultHandlerMock = {
|
|
61
|
+
handleResult: jest.fn().mockResolvedValue(undefined),
|
|
62
|
+
} as unknown as ResultHandler;
|
|
63
|
+
|
|
64
|
+
service = new ExtractorService(sdkMock, callConfig, resultHandlerMock);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('should initialize and register interval', () => {
|
|
68
|
+
expect(service).toBeDefined();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
describe('extraction method', () => {
|
|
72
|
+
const runExtract = async (): Promise<void> => {
|
|
73
|
+
await expect(service.onRun()).resolves.not.toThrow();
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
it('should perform extraction without errors', async () => {
|
|
77
|
+
await runExtract();
|
|
78
|
+
|
|
79
|
+
expect(httpClientMock.request).toHaveBeenCalledWith(
|
|
80
|
+
'GET',
|
|
81
|
+
'compiled-https://localhost:443',
|
|
82
|
+
{ data: undefined, headers: { 'Content-Type': 'text' } },
|
|
83
|
+
);
|
|
84
|
+
expect(resultHandlerMock.handleResult).toHaveBeenCalledWith(
|
|
85
|
+
'some-result-data',
|
|
86
|
+
callConfig,
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
});
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CompileDelegate,
|
|
3
|
+
ConnectorSDKInterface,
|
|
4
|
+
HttpClientSDKInterface,
|
|
5
|
+
HttpMethod,
|
|
6
|
+
IntervalHandler,
|
|
7
|
+
} from '@transai/connector-runtime-sdk';
|
|
8
|
+
|
|
9
|
+
import { HttpClientAuthentication } from './http-client-authentication';
|
|
10
|
+
import { ResultHandler } from './result.handler';
|
|
11
|
+
import { ApiConfig, ConnectorConfig } from './types';
|
|
12
|
+
|
|
13
|
+
export class ExtractorService implements IntervalHandler {
|
|
14
|
+
readonly #sdk: ConnectorSDKInterface<ConnectorConfig>;
|
|
15
|
+
|
|
16
|
+
readonly #apiConfig: ApiConfig;
|
|
17
|
+
|
|
18
|
+
readonly #resultHandler: ResultHandler;
|
|
19
|
+
|
|
20
|
+
readonly #httpClient: HttpClientSDKInterface;
|
|
21
|
+
|
|
22
|
+
readonly #requestMethod: HttpMethod;
|
|
23
|
+
|
|
24
|
+
readonly #urlDelegate?: CompileDelegate;
|
|
25
|
+
|
|
26
|
+
readonly #bodyDelegate?: CompileDelegate;
|
|
27
|
+
|
|
28
|
+
constructor(
|
|
29
|
+
sdk: ConnectorSDKInterface<ConnectorConfig>,
|
|
30
|
+
apiConfig: ApiConfig,
|
|
31
|
+
resultHandler: ResultHandler,
|
|
32
|
+
httpClient?: HttpClientSDKInterface,
|
|
33
|
+
) {
|
|
34
|
+
this.#sdk = sdk;
|
|
35
|
+
this.#apiConfig = apiConfig;
|
|
36
|
+
this.#resultHandler = resultHandler;
|
|
37
|
+
|
|
38
|
+
const { config } = this.#sdk;
|
|
39
|
+
|
|
40
|
+
this.#requestMethod = this.#apiConfig.method ?? config.method ?? 'GET';
|
|
41
|
+
this.#urlDelegate = this.#apiConfig.url
|
|
42
|
+
? this.#sdk.templating.compile(this.#apiConfig.url)
|
|
43
|
+
: undefined;
|
|
44
|
+
this.#bodyDelegate = this.#apiConfig.body
|
|
45
|
+
? this.#sdk.templating.compile(this.#apiConfig.body)
|
|
46
|
+
: undefined;
|
|
47
|
+
|
|
48
|
+
if (this.#apiConfig.url) {
|
|
49
|
+
this.#httpClient = this.#sdk
|
|
50
|
+
.httpClient()
|
|
51
|
+
.setRequestOptionsFormatter(
|
|
52
|
+
HttpClientAuthentication.createForAuthConfig(
|
|
53
|
+
this.#apiConfig,
|
|
54
|
+
this.#sdk.httpClient(),
|
|
55
|
+
),
|
|
56
|
+
);
|
|
57
|
+
} else if (httpClient) {
|
|
58
|
+
this.#httpClient = httpClient;
|
|
59
|
+
} else {
|
|
60
|
+
throw new Error(
|
|
61
|
+
`No HTTP client or URL provided for API extractor: ${apiConfig.name}`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
get name(): string {
|
|
67
|
+
return `api-extractor-${this.#apiConfig.name}`;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async onRun(): Promise<void> {
|
|
71
|
+
const latestOffset = await this.#sdk.offsetStore.getOffset(
|
|
72
|
+
`${this.#apiConfig.offsetFilePrefix ?? 'offset'}_${this.#apiConfig.name}`,
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
this.#sdk.logger.verbose(`[API] [${this.#apiConfig.name}] Executing query`);
|
|
76
|
+
|
|
77
|
+
const compileOptions = {
|
|
78
|
+
...latestOffset,
|
|
79
|
+
limit: this.#apiConfig.batchSize ?? 10,
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const result = await this.#httpClient.request<string>(
|
|
83
|
+
this.#requestMethod,
|
|
84
|
+
this.#urlDelegate?.(compileOptions) ?? '',
|
|
85
|
+
{
|
|
86
|
+
data: this.#bodyDelegate?.(compileOptions),
|
|
87
|
+
headers: {
|
|
88
|
+
'Content-Type': this.#apiConfig.format ?? 'text',
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
await this.#resultHandler.handleResult(result.data, this.#apiConfig);
|
|
94
|
+
|
|
95
|
+
this.#sdk.logger.debug(`[API] [${this.#apiConfig.name}] Ran query`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import {
|
|
2
|
+
HttpClientSDKInterface,
|
|
3
|
+
HttpRequestOptions,
|
|
4
|
+
HttpRequestOptionsFormatter,
|
|
5
|
+
} from '@transai/connector-runtime-sdk';
|
|
6
|
+
|
|
7
|
+
import { HttpClientAuthConfig } from './types';
|
|
8
|
+
|
|
9
|
+
export class HttpClientAuthentication {
|
|
10
|
+
readonly #config: HttpClientAuthConfig;
|
|
11
|
+
|
|
12
|
+
readonly #tokenHttpClient: HttpClientSDKInterface;
|
|
13
|
+
|
|
14
|
+
#sessionToken?: {
|
|
15
|
+
token: string;
|
|
16
|
+
expiresAt: number;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
private constructor(
|
|
20
|
+
config: HttpClientAuthConfig,
|
|
21
|
+
tokenHttpClient: HttpClientSDKInterface,
|
|
22
|
+
) {
|
|
23
|
+
this.#config = config;
|
|
24
|
+
this.#tokenHttpClient = tokenHttpClient;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
static createForAuthConfig(
|
|
28
|
+
config: HttpClientAuthConfig,
|
|
29
|
+
tokenHttpClient: HttpClientSDKInterface,
|
|
30
|
+
): HttpRequestOptionsFormatter {
|
|
31
|
+
const instance = new HttpClientAuthentication(config, tokenHttpClient);
|
|
32
|
+
|
|
33
|
+
return instance.format.bind(instance);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async format<D = string | object>(
|
|
37
|
+
requestOptions: HttpRequestOptions<D>,
|
|
38
|
+
): Promise<HttpRequestOptions<D>> {
|
|
39
|
+
if (this.#config.authorization) {
|
|
40
|
+
return {
|
|
41
|
+
...requestOptions,
|
|
42
|
+
headers: {
|
|
43
|
+
...requestOptions.headers,
|
|
44
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
45
|
+
Authorization: this.#config.authorization!,
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (
|
|
51
|
+
this.#config.tokenUrl &&
|
|
52
|
+
this.#config.clientId &&
|
|
53
|
+
this.#config.clientSecret
|
|
54
|
+
) {
|
|
55
|
+
return {
|
|
56
|
+
...requestOptions,
|
|
57
|
+
headers: {
|
|
58
|
+
...requestOptions.headers,
|
|
59
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
60
|
+
Authorization: await this.#formatTokenAuth(),
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return requestOptions;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async #formatTokenAuth(): Promise<string> {
|
|
69
|
+
const token = this.#sessionToken;
|
|
70
|
+
if (token && token.expiresAt > Date.now()) {
|
|
71
|
+
return token.token;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const response = await this.#tokenHttpClient.post<{
|
|
75
|
+
access_token: string;
|
|
76
|
+
expires_in: number;
|
|
77
|
+
}>(
|
|
78
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
79
|
+
this.#config.tokenUrl!,
|
|
80
|
+
{
|
|
81
|
+
grant_type: 'client_credentials',
|
|
82
|
+
client_id: this.#config.clientId,
|
|
83
|
+
client_secret: this.#config.clientSecret,
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
headers: {
|
|
87
|
+
Accept: 'application/json',
|
|
88
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
if (!response) {
|
|
94
|
+
throw new Error(
|
|
95
|
+
'Failed to authenticate and retrieve authentication token',
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const { access_token: accessToken, expires_in: expiresIn } = response.data;
|
|
100
|
+
|
|
101
|
+
this.#sessionToken = {
|
|
102
|
+
token: accessToken,
|
|
103
|
+
expiresAt: Date.now() + expiresIn * 1000 - 1000,
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
return accessToken;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { ConnectorSDKInterface } from '@transai/connector-runtime-sdk';
|
|
2
|
+
|
|
3
|
+
import { ResultHandler } from './result.handler';
|
|
4
|
+
import { ApiConfig, ConnectorConfig } from './types';
|
|
5
|
+
|
|
6
|
+
describe('ResultHandler', () => {
|
|
7
|
+
let resultHandler: ResultHandler;
|
|
8
|
+
|
|
9
|
+
let sdkMock: ConnectorSDKInterface<ConnectorConfig>;
|
|
10
|
+
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
sdkMock = {
|
|
13
|
+
logger: {
|
|
14
|
+
info: jest.fn(),
|
|
15
|
+
debug: jest.fn(),
|
|
16
|
+
error: jest.fn(),
|
|
17
|
+
verbose: jest.fn(),
|
|
18
|
+
warn: jest.fn(),
|
|
19
|
+
},
|
|
20
|
+
offsetStore: {
|
|
21
|
+
setOffset: jest.fn(),
|
|
22
|
+
},
|
|
23
|
+
config: {},
|
|
24
|
+
sender: {
|
|
25
|
+
documents: jest.fn().mockResolvedValue(true),
|
|
26
|
+
metricsLegacy: jest.fn().mockResolvedValue(true),
|
|
27
|
+
},
|
|
28
|
+
} as unknown as ConnectorSDKInterface<ConnectorConfig>;
|
|
29
|
+
|
|
30
|
+
resultHandler = new ResultHandler(sdkMock);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it('should be defined', () => {
|
|
34
|
+
expect(resultHandler).toBeDefined();
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe('handle batch data', () => {
|
|
38
|
+
const apiConfig = {
|
|
39
|
+
name: 'test-api',
|
|
40
|
+
listField: 'items',
|
|
41
|
+
incrementalField: 'updatedAt',
|
|
42
|
+
offsetFilePrefix: 'offset_test',
|
|
43
|
+
keyField: 'id',
|
|
44
|
+
} as ApiConfig;
|
|
45
|
+
|
|
46
|
+
const result = JSON.stringify({
|
|
47
|
+
items: [
|
|
48
|
+
{ id: 1, updatedAt: '2024-01-01T00:00:00Z' },
|
|
49
|
+
{ id: 2, updatedAt: '2024-01-02T00:00:00Z' },
|
|
50
|
+
],
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('should send batch data and set offset', async () => {
|
|
54
|
+
await resultHandler.handleResult(result, apiConfig);
|
|
55
|
+
|
|
56
|
+
expect(sdkMock.sender.documents).toHaveBeenCalledWith(
|
|
57
|
+
[
|
|
58
|
+
{ id: 1, updatedAt: '2024-01-01T00:00:00Z' },
|
|
59
|
+
{ id: 2, updatedAt: '2024-01-02T00:00:00Z' },
|
|
60
|
+
],
|
|
61
|
+
{
|
|
62
|
+
collection: 'api_test-api',
|
|
63
|
+
incrementalField: 'updatedAt',
|
|
64
|
+
keyField: 'id',
|
|
65
|
+
},
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
expect(sdkMock.offsetStore.setOffset).toHaveBeenCalledWith(
|
|
69
|
+
expect.objectContaining({
|
|
70
|
+
timestamp: new Date('2024-01-02T00:00:00Z').getTime(),
|
|
71
|
+
}),
|
|
72
|
+
'offset_test_test-api',
|
|
73
|
+
);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('should send batch data as metrics', async () => {
|
|
77
|
+
await resultHandler.handleResult(result, {
|
|
78
|
+
...apiConfig,
|
|
79
|
+
type: 'metric',
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
expect(sdkMock.sender.metricsLegacy).toHaveBeenCalledWith(
|
|
83
|
+
[
|
|
84
|
+
{ id: 1, updatedAt: '2024-01-01T00:00:00Z' },
|
|
85
|
+
{ id: 2, updatedAt: '2024-01-02T00:00:00Z' },
|
|
86
|
+
],
|
|
87
|
+
{
|
|
88
|
+
collection: 'api_test-api',
|
|
89
|
+
incrementalField: 'updatedAt',
|
|
90
|
+
keyField: 'id',
|
|
91
|
+
},
|
|
92
|
+
);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe('handle single record', () => {
|
|
97
|
+
const apiConfig = {
|
|
98
|
+
name: 'test-api',
|
|
99
|
+
} as ApiConfig;
|
|
100
|
+
|
|
101
|
+
const result = JSON.stringify({ id: 1, updatedAt: '2024-01-01T00:00:00Z' });
|
|
102
|
+
|
|
103
|
+
it('should send single record and set offset', async () => {
|
|
104
|
+
await resultHandler.handleResult(result, apiConfig);
|
|
105
|
+
|
|
106
|
+
expect(sdkMock.sender.documents).toHaveBeenCalledWith(
|
|
107
|
+
[{ id: 1, updatedAt: '2024-01-01T00:00:00Z' }],
|
|
108
|
+
{
|
|
109
|
+
collection: 'api_test-api',
|
|
110
|
+
},
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
expect(sdkMock.offsetStore.setOffset).toHaveBeenCalledWith(
|
|
114
|
+
expect.objectContaining({
|
|
115
|
+
timestamp: expect.any(Number),
|
|
116
|
+
}),
|
|
117
|
+
'offset_test-api',
|
|
118
|
+
);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('should send single record as metric', async () => {
|
|
122
|
+
await resultHandler.handleResult(result, {
|
|
123
|
+
...apiConfig,
|
|
124
|
+
type: 'metric',
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
expect(sdkMock.sender.metricsLegacy).toHaveBeenCalledWith(
|
|
128
|
+
[{ id: 1, updatedAt: '2024-01-01T00:00:00Z' }],
|
|
129
|
+
{
|
|
130
|
+
collection: 'api_test-api',
|
|
131
|
+
},
|
|
132
|
+
);
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
});
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { ConnectorSDKInterface } from '@transai/connector-runtime-sdk';
|
|
2
|
+
import { decode } from 'html-entities';
|
|
3
|
+
|
|
4
|
+
import { ApiConfig, ConnectorConfig } from './types';
|
|
5
|
+
|
|
6
|
+
export class ResultHandler {
|
|
7
|
+
readonly #sdk: ConnectorSDKInterface<ConnectorConfig>;
|
|
8
|
+
|
|
9
|
+
constructor(sdk: ConnectorSDKInterface<ConnectorConfig>) {
|
|
10
|
+
this.#sdk = sdk;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
async handleResult(result: string, apiConfig: ApiConfig): Promise<void> {
|
|
14
|
+
const parsedContent = JSON.parse(decode(result));
|
|
15
|
+
const keys = Object.keys(parsedContent);
|
|
16
|
+
if (keys.length === 0) {
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
if (apiConfig.listField) {
|
|
21
|
+
await this.#sendBatch(parsedContent, apiConfig);
|
|
22
|
+
} else {
|
|
23
|
+
await this.#sendSingleRecord(parsedContent, apiConfig);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async #sendBatch(
|
|
28
|
+
parsedContent: Record<string, Array<{ [key: string]: string }>>,
|
|
29
|
+
apiConfig: ApiConfig,
|
|
30
|
+
): Promise<void> {
|
|
31
|
+
const list = parsedContent[apiConfig.listField ?? ''];
|
|
32
|
+
if (!(list && Array.isArray(list))) {
|
|
33
|
+
this.#sdk.logger.debug(
|
|
34
|
+
`[API] [${apiConfig.name}] No records found, skipping. ${JSON.stringify(list)}`,
|
|
35
|
+
);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
await this.#sendData(list, apiConfig);
|
|
40
|
+
|
|
41
|
+
const item = list[list.length - 1];
|
|
42
|
+
this.#sdk.offsetStore.setOffset(
|
|
43
|
+
{
|
|
44
|
+
timestamp: (apiConfig.incrementalField
|
|
45
|
+
? new Date(item[apiConfig.incrementalField] as string)
|
|
46
|
+
: new Date()
|
|
47
|
+
).getTime(),
|
|
48
|
+
id: 0,
|
|
49
|
+
rawTimestamp: 0,
|
|
50
|
+
},
|
|
51
|
+
`${apiConfig.offsetFilePrefix ?? 'offset'}_${apiConfig.name}`,
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async #sendSingleRecord(
|
|
56
|
+
parsedContent: object,
|
|
57
|
+
apiConfig: ApiConfig,
|
|
58
|
+
): Promise<void> {
|
|
59
|
+
const success = await this.#sendData([parsedContent], apiConfig);
|
|
60
|
+
if (!success) {
|
|
61
|
+
this.#sdk.logger.debug(
|
|
62
|
+
`[API] [${apiConfig.name}] Error while sending record to Kafka: `,
|
|
63
|
+
parsedContent,
|
|
64
|
+
);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
this.#sdk.offsetStore.setOffset(
|
|
69
|
+
{
|
|
70
|
+
timestamp: Date.now(),
|
|
71
|
+
id: 0,
|
|
72
|
+
rawTimestamp: 0,
|
|
73
|
+
},
|
|
74
|
+
`${apiConfig.offsetFilePrefix ?? 'offset'}_${apiConfig.name}`,
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async #sendData(list: Array<object>, apiConfig: ApiConfig): Promise<boolean> {
|
|
79
|
+
const collection = `${this.#sdk.config.datasourceIdentifier ?? 'api'}_${apiConfig.name}`;
|
|
80
|
+
const metadata = {
|
|
81
|
+
...(apiConfig.metadata ?? {}),
|
|
82
|
+
...(apiConfig.incrementalField
|
|
83
|
+
? { incrementalField: apiConfig.incrementalField }
|
|
84
|
+
: {}),
|
|
85
|
+
...(apiConfig.keyField ? { keyField: apiConfig.keyField } : {}),
|
|
86
|
+
collection,
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
if (apiConfig.type === 'metric') {
|
|
90
|
+
return (
|
|
91
|
+
(await this.#sdk.sender.metricsLegacy(
|
|
92
|
+
list as Array<never>,
|
|
93
|
+
metadata,
|
|
94
|
+
)) === true
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return (await this.#sdk.sender.documents(list, metadata)) === true;
|
|
99
|
+
}
|
|
100
|
+
}
|
package/src/lib/types.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { HttpMethod } from '@transai/connector-runtime-sdk';
|
|
2
|
+
import { BaseConnectorConfig } from '@xip-online-data/types';
|
|
3
|
+
|
|
4
|
+
export interface HttpClientConfig {
|
|
5
|
+
method?: HttpMethod;
|
|
6
|
+
url?: string;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface HttpClientAuthConfig {
|
|
10
|
+
tokenUrl?: string;
|
|
11
|
+
clientId?: string;
|
|
12
|
+
clientSecret?: string;
|
|
13
|
+
authorization?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface ApiConfig extends HttpClientConfig, HttpClientAuthConfig {
|
|
17
|
+
name: string;
|
|
18
|
+
interval: number;
|
|
19
|
+
|
|
20
|
+
body?: string;
|
|
21
|
+
batchSize?: number;
|
|
22
|
+
offsetFilePrefix?: string;
|
|
23
|
+
format?: string;
|
|
24
|
+
listField?: string;
|
|
25
|
+
keyField?: string;
|
|
26
|
+
incrementalField?: string;
|
|
27
|
+
type?: 'metric' | 'document';
|
|
28
|
+
metadata?: {
|
|
29
|
+
[key: string]: string;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface ConnectorConfig
|
|
34
|
+
extends BaseConnectorConfig,
|
|
35
|
+
HttpClientConfig,
|
|
36
|
+
HttpClientAuthConfig {
|
|
37
|
+
apiCalls?: Array<ApiConfig>;
|
|
38
|
+
}
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "../../tsconfig.base.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"module": "commonjs",
|
|
5
|
+
"forceConsistentCasingInFileNames": true,
|
|
6
|
+
"strict": true,
|
|
7
|
+
"noImplicitOverride": true,
|
|
8
|
+
"noImplicitReturns": true,
|
|
9
|
+
"noFallthroughCasesInSwitch": true,
|
|
10
|
+
"noPropertyAccessFromIndexSignature": true
|
|
11
|
+
},
|
|
12
|
+
"files": [],
|
|
13
|
+
"include": [],
|
|
14
|
+
"references": [
|
|
15
|
+
{
|
|
16
|
+
"path": "./tsconfig.lib.json"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"path": "./tsconfig.spec.json"
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "./tsconfig.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"outDir": "../../dist/out-tsc",
|
|
5
|
+
"module": "commonjs",
|
|
6
|
+
"types": ["jest", "node"]
|
|
7
|
+
},
|
|
8
|
+
"include": [
|
|
9
|
+
"jest.config.ts",
|
|
10
|
+
"src/**/*.test.ts",
|
|
11
|
+
"src/**/*.spec.ts",
|
|
12
|
+
"src/**/*.d.ts"
|
|
13
|
+
]
|
|
14
|
+
}
|