@transai/connector-runner-api 0.1.0 → 0.3.0

package/libs/types/src/lib/types.d.ts

@@ -0,0 +1,116 @@
+export interface MessageMonitorConfig {
+    type: 'in-memory' | 'redis' | 'disabled';
+}
+export declare enum ConfiguredConnectorTypes {
+    API = "api",
+    API_SINK = "api-sink",
+    API_SOURCE = "api-source",
+    DUMMY_NODE = "dummy-node",
+    FILE_COPY = "file-copy",
+    FILE = "file",
+    FILE_SINK = "file-sink",
+    FILE_SOURCE = "file-source",
+    SAMBA_SINK = "samba-sink",
+    SAMBA_SOURCE = "samba-source",
+    SQL_SINK = "sql-sink",
+    SQL_SOURCE = "sql-source",
+    MQTT = "mqtt",
+    CUBE_QUERY_RUNNER = "cube-query-runner",
+    FACTORY_NEBULA_SOURCE = "factory-nebula-source",
+    BYSTRONIC = "bystronic",
+    IMAP_SOURCE = "imap-source",
+    IMAP_SINK = "imap-sink",
+    OPCUA_SOURCE = "opcua-source",
+    OPCUA_SINK = "opcua-sink",
+    AI_AGENT = "ai-agent",
+    MICROSOFT_OFFICE365_EMAIL = "microsoft-office365-email",
+    MKG = "mkg"
+}
+export interface ConnectorConfig {
+    type: ConfiguredConnectorTypes;
+    identifier: string;
+}
+export interface RedisMessageMonitorConfig extends MessageMonitorConfig {
+    type: 'redis';
+    url: string;
+    ttl?: number;
+}
+export interface InMemoryMessageMonitorConfig extends MessageMonitorConfig {
+    type: 'in-memory';
+    messagesToKeep: number;
+}
+export interface DisabledMessageMonitorConfig extends MessageMonitorConfig {
+    type: 'disabled';
+}
+export interface TopicRegex {
+    pattern: string;
+    flags: string;
+}
+export interface FileRegex {
+    pattern: string;
+    flags: string;
+}
+export declare function isTopicRegex(obj: TopicRegex | string): obj is TopicRegex;
+export interface CubeConfigHeaders {
+    Authorization: string;
+}
+export interface CubeConfig {
+    auth0_client_secret?: string;
+    auth0_client_id?: string;
+    auth0_audience?: string;
+    auth0_token_url?: string;
+    authorization?: string;
+    apiUrl: string;
+    headers?: CubeConfigHeaders;
+    redis_url?: string;
+}
+export declare function isCubeConfigType(obj: CubeConfig): obj is CubeConfig;
+export interface AwsKafkaSaslConfig {
+    region?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    mechanism: 'aws';
+}
+export declare function isAwsKafkaSalsConfig(obj: AwsKafkaSaslConfig): obj is AwsKafkaSaslConfig;
+export interface DatabaseConfig {
+    uri: string;
+    ssl?: boolean;
+    tenantPassword?: string;
+}
+export interface ActionConfig {
+    timeSensitive: boolean;
+}
+export interface KafkaBrokerConfig {
+    groupId: string;
+    clientId: string;
+    brokers: Array<string>;
+    consumerTopics?: Array<string | TopicRegex>;
+    sasl?: AwsKafkaSaslConfig;
+    intervalCheckForNewTopics?: number;
+    disableLogs?: boolean;
+    autoCommitThreshold?: number;
+    autoCommitInterval?: number;
+    partitionsConsumedConcurrently?: number;
+    messageMonitor?: RedisMessageMonitorConfig | InMemoryMessageMonitorConfig | DisabledMessageMonitorConfig;
+    useConfluentLibrary?: boolean;
+    newConsumerProtocol?: boolean;
+}
+export declare function isKafkaBrokerConfigType(obj: KafkaBrokerConfig): obj is KafkaBrokerConfig;
+export declare function isActionConfigType(obj: any): obj is ActionConfig;
+export declare function isDatabaseConfigType(obj: DatabaseConfig): obj is DatabaseConfig;
+export interface BaseConnectorConfig {
+    processIdentifier: string;
+    tenantIdentifier: string;
+    datasourceIdentifier: string;
+    kafka: KafkaBrokerConfig;
+    action?: ActionConfig;
+}
+export declare function isBaseConnectorConfigType(obj: BaseConnectorConfig): obj is BaseConnectorConfig;
+export interface AiAgentConfig {
+    openai: {
+        apiKey: string;
+        model: 'gpt-3.5-turbo' | 'gpt-4' | 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini' | 'o1-preview' | 'o1-mini';
+        temperature?: number;
+        timeout?: number;
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transai/connector-runner-api",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "publishConfig": {
     "access": "public"
   },

package/.eslintrc.json

@@ -1,18 +0,0 @@
-{
-  "extends": ["../../.eslintrc.json"],
-  "ignorePatterns": ["!**/*"],
-  "overrides": [
-    {
-      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
-      "rules": {}
-    },
-    {
-      "files": ["*.ts", "*.tsx"],
-      "rules": {}
-    },
-    {
-      "files": ["*.js", "*.jsx"],
-      "rules": {}
-    }
-  ]
-}

package/jest.config.ts

@@ -1,10 +0,0 @@
-export default {
-  displayName: 'connector-runner-api',
-  preset: '../../jest.preset.js',
-  testEnvironment: 'node',
-  transform: {
-    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
-  },
-  moduleFileExtensions: ['ts', 'js', 'html'],
-  coverageDirectory: '../../coverage/libs/connector-runner-api',
-};

package/project.json

@@ -1,29 +0,0 @@
-{
-  "name": "connector-runner-api",
-  "$schema": "../../node_modules/nx/schemas/project-schema.json",
-  "sourceRoot": "libs/connector-runner-api/src",
-  "projectType": "library",
-  "tags": [
-    "connector:runner",
-    "connector:runner-sdk",
-    "connector:api",
-    "connector:source",
-    "provider:@transai"
-  ],
-  "targets": {
-    "build": {
-      "executor": "@transai/tools:connector",
-      "outputs": ["{options.outputPath}"]
-    },
-    "lint": {
-      "executor": "@nx/eslint:lint"
-    },
-    "test": {
-      "executor": "@nx/jest:jest",
-      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
-      "options": {
-        "jestConfig": "libs/connector-runner-api/jest.config.ts"
-      }
-    }
-  }
-}

package/src/lib/connector-runner-api.ts

@@ -1,53 +0,0 @@
-import {
-  ConnectorRuntimeSDK,
-  ConnectorSDKInterface,
-  HttpClientSDKInterface,
-} from '@transai/connector-runtime-sdk';
-import { ConnectorInterface } from '@xip-online-data/types';
-
-import { ExtractorService } from './extractor.service';
-import { HttpClientAuthentication } from './http-client-authentication';
-import { ResultHandler } from './result.handler';
-import { ConnectorConfig } from './types';
-
-export class ConnectorRunnerApi extends ConnectorRuntimeSDK<ConnectorConfig> {
-  readonly #httpClient?: HttpClientSDKInterface = undefined;
-
-  constructor(
-    connector: ConnectorInterface,
-    connectorSDK: ConnectorSDKInterface,
-  ) {
-    super(connector, connectorSDK);
-
-    const { config } = this.connectorSDK;
-    if (config.url) {
-      this.#httpClient = this.connectorSDK
-        .httpClient({
-          baseUrl: config.url,
-        })
-        .setRequestOptionsFormatter(
-          HttpClientAuthentication.createForAuthConfig(
-            config,
-            this.connectorSDK.httpClient(),
-          ),
-        );
-    }
-  }
-
-  override init = async (): Promise<void> => {
-    const { config } = this.connectorSDK;
-    const resultHandler = new ResultHandler(this.connectorSDK);
-
-    (config.apiCalls ?? []).forEach((apiConfig) => {
-      this.connectorSDK.processing.registerInterval(
-        apiConfig.interval,
-        new ExtractorService(
-          this.connectorSDK,
-          apiConfig,
-          resultHandler,
-          this.#httpClient,
-        ),
-      );
-    });
-  };
-}

package/src/lib/extractor.service.spec.ts

@@ -1,90 +0,0 @@
-import {
-  CompileDelegate,
-  ConnectorSDKInterface,
-  HttpClientSDKInterface,
-} from '@transai/connector-runtime-sdk';
-
-import { ExtractorService } from './extractor.service';
-import { ResultHandler } from './result.handler';
-import { ConnectorConfig } from './types';
-
-describe('ExtractorService', () => {
-  let service: ExtractorService;
-
-  const callConfig = {
-    name: 'TestCall',
-    interval: 60,
-    offsetFilePrefix: 'test_offset',
-    url: 'https://localhost:443',
-  };
-
-  const httpClientMock = {
-    request: jest.fn().mockResolvedValue({
-      status: 200,
-      data: 'some-result-data',
-    }),
-    setRequestOptionsFormatter: jest.fn().mockReturnThis(),
-  } as unknown as HttpClientSDKInterface;
-  let sdkMock: ConnectorSDKInterface<ConnectorConfig>;
-  let resultHandlerMock: ResultHandler;
-
-  beforeEach(() => {
-    sdkMock = {
-      logger: {
-        info: jest.fn(),
-        debug: jest.fn(),
-        error: jest.fn(),
-        verbose: jest.fn(),
-        warn: jest.fn(),
-      },
-      offsetStore: {
-        getOffset: jest.fn().mockResolvedValue({
-          id: 0,
-          timestamp: new Date().getTime(),
-          rawTimestamp: new Date().toISOString(),
-          isoDate: new Date().toISOString(),
-        }),
-      },
-      templating: {
-        compile: jest
-          .fn()
-          .mockImplementation(
-            (input) =>
-              jest.fn().mockReturnValue(`compiled-${input}`) as CompileDelegate,
-          ),
-      },
-      config: {},
-      httpClient: jest.fn().mockReturnValue(httpClientMock),
-    } as unknown as ConnectorSDKInterface<ConnectorConfig>;
-
-    resultHandlerMock = {
-      handleResult: jest.fn().mockResolvedValue(undefined),
-    } as unknown as ResultHandler;
-
-    service = new ExtractorService(sdkMock, callConfig, resultHandlerMock);
-  });
-
-  it('should initialize and register interval', () => {
-    expect(service).toBeDefined();
-  });
-
-  describe('extraction method', () => {
-    const runExtract = async (): Promise<void> => {
-      await expect(service.onRun()).resolves.not.toThrow();
-    };
-
-    it('should perform extraction without errors', async () => {
-      await runExtract();
-
-      expect(httpClientMock.request).toHaveBeenCalledWith(
-        'GET',
-        'compiled-https://localhost:443',
-        { data: undefined, headers: { 'Content-Type': 'text' } },
-      );
-      expect(resultHandlerMock.handleResult).toHaveBeenCalledWith(
-        'some-result-data',
-        callConfig,
-      );
-    });
-  });
-});

package/src/lib/extractor.service.ts

@@ -1,97 +0,0 @@
-import {
-  CompileDelegate,
-  ConnectorSDKInterface,
-  HttpClientSDKInterface,
-  HttpMethod,
-  IntervalHandler,
-} from '@transai/connector-runtime-sdk';
-
-import { HttpClientAuthentication } from './http-client-authentication';
-import { ResultHandler } from './result.handler';
-import { ApiConfig, ConnectorConfig } from './types';
-
-export class ExtractorService implements IntervalHandler {
-  readonly #sdk: ConnectorSDKInterface<ConnectorConfig>;
-
-  readonly #apiConfig: ApiConfig;
-
-  readonly #resultHandler: ResultHandler;
-
-  readonly #httpClient: HttpClientSDKInterface;
-
-  readonly #requestMethod: HttpMethod;
-
-  readonly #urlDelegate?: CompileDelegate;
-
-  readonly #bodyDelegate?: CompileDelegate;
-
-  constructor(
-    sdk: ConnectorSDKInterface<ConnectorConfig>,
-    apiConfig: ApiConfig,
-    resultHandler: ResultHandler,
-    httpClient?: HttpClientSDKInterface,
-  ) {
-    this.#sdk = sdk;
-    this.#apiConfig = apiConfig;
-    this.#resultHandler = resultHandler;
-
-    const { config } = this.#sdk;
-
-    this.#requestMethod = this.#apiConfig.method ?? config.method ?? 'GET';
-    this.#urlDelegate = this.#apiConfig.url
-      ? this.#sdk.templating.compile(this.#apiConfig.url)
-      : undefined;
-    this.#bodyDelegate = this.#apiConfig.body
-      ? this.#sdk.templating.compile(this.#apiConfig.body)
-      : undefined;
-
-    if (this.#apiConfig.url) {
-      this.#httpClient = this.#sdk
-        .httpClient()
-        .setRequestOptionsFormatter(
-          HttpClientAuthentication.createForAuthConfig(
-            this.#apiConfig,
-            this.#sdk.httpClient(),
-          ),
-        );
-    } else if (httpClient) {
-      this.#httpClient = httpClient;
-    } else {
-      throw new Error(
-        `No HTTP client or URL provided for API extractor: ${apiConfig.name}`,
-      );
-    }
-  }
-
-  get name(): string {
-    return `api-extractor-${this.#apiConfig.name}`;
-  }
-
-  async onRun(): Promise<void> {
-    const latestOffset = await this.#sdk.offsetStore.getOffset(
-      `${this.#apiConfig.offsetFilePrefix ?? 'offset'}_${this.#apiConfig.name}`,
-    );
-
-    this.#sdk.logger.verbose(`[API] [${this.#apiConfig.name}] Executing query`);
-
-    const compileOptions = {
-      ...latestOffset,
-      limit: this.#apiConfig.batchSize ?? 10,
-    };
-
-    const result = await this.#httpClient.request<string>(
-      this.#requestMethod,
-      this.#urlDelegate?.(compileOptions) ?? '',
-      {
-        data: this.#bodyDelegate?.(compileOptions),
-        headers: {
-          'Content-Type': this.#apiConfig.format ?? 'text',
-        },
-      },
-    );
-
-    await this.#resultHandler.handleResult(result.data, this.#apiConfig);
-
-    this.#sdk.logger.debug(`[API] [${this.#apiConfig.name}] Ran query`);
-  }
-}

package/src/lib/http-client-authentication.ts

@@ -1,108 +0,0 @@
-import {
-  HttpClientSDKInterface,
-  HttpRequestOptions,
-  HttpRequestOptionsFormatter,
-} from '@transai/connector-runtime-sdk';
-
-import { HttpClientAuthConfig } from './types';
-
-export class HttpClientAuthentication {
-  readonly #config: HttpClientAuthConfig;
-
-  readonly #tokenHttpClient: HttpClientSDKInterface;
-
-  #sessionToken?: {
-    token: string;
-    expiresAt: number;
-  };
-
-  private constructor(
-    config: HttpClientAuthConfig,
-    tokenHttpClient: HttpClientSDKInterface,
-  ) {
-    this.#config = config;
-    this.#tokenHttpClient = tokenHttpClient;
-  }
-
-  static createForAuthConfig(
-    config: HttpClientAuthConfig,
-    tokenHttpClient: HttpClientSDKInterface,
-  ): HttpRequestOptionsFormatter {
-    const instance = new HttpClientAuthentication(config, tokenHttpClient);
-
-    return instance.format.bind(instance);
-  }
-
-  async format<D = string | object>(
-    requestOptions: HttpRequestOptions<D>,
-  ): Promise<HttpRequestOptions<D>> {
-    if (this.#config.authorization) {
-      return {
-        ...requestOptions,
-        headers: {
-          ...requestOptions.headers,
-          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-          Authorization: this.#config.authorization!,
-        },
-      };
-    }
-
-    if (
-      this.#config.tokenUrl &&
-      this.#config.clientId &&
-      this.#config.clientSecret
-    ) {
-      return {
-        ...requestOptions,
-        headers: {
-          ...requestOptions.headers,
-          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-          Authorization: await this.#formatTokenAuth(),
-        },
-      };
-    }
-
-    return requestOptions;
-  }
-
-  async #formatTokenAuth(): Promise<string> {
-    const token = this.#sessionToken;
-    if (token && token.expiresAt > Date.now()) {
-      return token.token;
-    }
-
-    const response = await this.#tokenHttpClient.post<{
-      access_token: string;
-      expires_in: number;
-    }>(
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      this.#config.tokenUrl!,
-      {
-        grant_type: 'client_credentials',
-        client_id: this.#config.clientId,
-        client_secret: this.#config.clientSecret,
-      },
-      {
-        headers: {
-          Accept: 'application/json',
-          'Content-Type': 'application/x-www-form-urlencoded',
-        },
-      },
-    );
-
-    if (!response) {
-      throw new Error(
-        'Failed to authenticate and retrieve authentication token',
-      );
-    }
-
-    const { access_token: accessToken, expires_in: expiresIn } = response.data;
-
-    this.#sessionToken = {
-      token: accessToken,
-      expiresAt: Date.now() + expiresIn * 1000 - 1000,
-    };
-
-    return accessToken;
-  }
-}

package/src/lib/result.handler.spec.ts

@@ -1,135 +0,0 @@
-import { ConnectorSDKInterface } from '@transai/connector-runtime-sdk';
-
-import { ResultHandler } from './result.handler';
-import { ApiConfig, ConnectorConfig } from './types';
-
-describe('ResultHandler', () => {
-  let resultHandler: ResultHandler;
-
-  let sdkMock: ConnectorSDKInterface<ConnectorConfig>;
-
-  beforeEach(() => {
-    sdkMock = {
-      logger: {
-        info: jest.fn(),
-        debug: jest.fn(),
-        error: jest.fn(),
-        verbose: jest.fn(),
-        warn: jest.fn(),
-      },
-      offsetStore: {
-        setOffset: jest.fn(),
-      },
-      config: {},
-      sender: {
-        documents: jest.fn().mockResolvedValue(true),
-        metricsLegacy: jest.fn().mockResolvedValue(true),
-      },
-    } as unknown as ConnectorSDKInterface<ConnectorConfig>;
-
-    resultHandler = new ResultHandler(sdkMock);
-  });
-
-  it('should be defined', () => {
-    expect(resultHandler).toBeDefined();
-  });
-
-  describe('handle batch data', () => {
-    const apiConfig = {
-      name: 'test-api',
-      listField: 'items',
-      incrementalField: 'updatedAt',
-      offsetFilePrefix: 'offset_test',
-      keyField: 'id',
-    } as ApiConfig;
-
-    const result = JSON.stringify({
-      items: [
-        { id: 1, updatedAt: '2024-01-01T00:00:00Z' },
-        { id: 2, updatedAt: '2024-01-02T00:00:00Z' },
-      ],
-    });
-
-    it('should send batch data and set offset', async () => {
-      await resultHandler.handleResult(result, apiConfig);
-
-      expect(sdkMock.sender.documents).toHaveBeenCalledWith(
-        [
-          { id: 1, updatedAt: '2024-01-01T00:00:00Z' },
-          { id: 2, updatedAt: '2024-01-02T00:00:00Z' },
-        ],
-        {
-          collection: 'api_test-api',
-          incrementalField: 'updatedAt',
-          keyField: 'id',
-        },
-      );
-
-      expect(sdkMock.offsetStore.setOffset).toHaveBeenCalledWith(
-        expect.objectContaining({
-          timestamp: new Date('2024-01-02T00:00:00Z').getTime(),
-        }),
-        'offset_test_test-api',
-      );
-    });
-
-    it('should send batch data as metrics', async () => {
-      await resultHandler.handleResult(result, {
-        ...apiConfig,
-        type: 'metric',
-      });
-
-      expect(sdkMock.sender.metricsLegacy).toHaveBeenCalledWith(
-        [
-          { id: 1, updatedAt: '2024-01-01T00:00:00Z' },
-          { id: 2, updatedAt: '2024-01-02T00:00:00Z' },
-        ],
-        {
-          collection: 'api_test-api',
-          incrementalField: 'updatedAt',
-          keyField: 'id',
-        },
-      );
-    });
-  });
-
-  describe('handle single record', () => {
-    const apiConfig = {
-      name: 'test-api',
-    } as ApiConfig;
-
-    const result = JSON.stringify({ id: 1, updatedAt: '2024-01-01T00:00:00Z' });
-
-    it('should send single record and set offset', async () => {
-      await resultHandler.handleResult(result, apiConfig);
-
-      expect(sdkMock.sender.documents).toHaveBeenCalledWith(
-        [{ id: 1, updatedAt: '2024-01-01T00:00:00Z' }],
-        {
-          collection: 'api_test-api',
-        },
-      );
-
-      expect(sdkMock.offsetStore.setOffset).toHaveBeenCalledWith(
-        expect.objectContaining({
-          timestamp: expect.any(Number),
-        }),
-        'offset_test-api',
-      );
-    });
-
-    it('should send single record as metric', async () => {
-      await resultHandler.handleResult(result, {
-        ...apiConfig,
-        type: 'metric',
-      });
-
-      expect(sdkMock.sender.metricsLegacy).toHaveBeenCalledWith(
-        [{ id: 1, updatedAt: '2024-01-01T00:00:00Z' }],
-        {
-          collection: 'api_test-api',
-        },
-      );
-    });
-  });
-});