@transai/connector-runner-file 0.1.0 → 0.2.0

package/libs/types/src/lib/types.d.ts

@@ -0,0 +1,116 @@
+export interface MessageMonitorConfig {
+    type: 'in-memory' | 'redis' | 'disabled';
+}
+export declare enum ConfiguredConnectorTypes {
+    API = "api",
+    API_SINK = "api-sink",
+    API_SOURCE = "api-source",
+    DUMMY_NODE = "dummy-node",
+    FILE_COPY = "file-copy",
+    FILE = "file",
+    FILE_SINK = "file-sink",
+    FILE_SOURCE = "file-source",
+    SAMBA_SINK = "samba-sink",
+    SAMBA_SOURCE = "samba-source",
+    SQL_SINK = "sql-sink",
+    SQL_SOURCE = "sql-source",
+    MQTT = "mqtt",
+    CUBE_QUERY_RUNNER = "cube-query-runner",
+    FACTORY_NEBULA_SOURCE = "factory-nebula-source",
+    BYSTRONIC = "bystronic",
+    IMAP_SOURCE = "imap-source",
+    IMAP_SINK = "imap-sink",
+    OPCUA_SOURCE = "opcua-source",
+    OPCUA_SINK = "opcua-sink",
+    AI_AGENT = "ai-agent",
+    MICROSOFT_OFFICE365_EMAIL = "microsoft-office365-email",
+    MKG = "mkg"
+}
+export interface ConnectorConfig {
+    type: ConfiguredConnectorTypes;
+    identifier: string;
+}
+export interface RedisMessageMonitorConfig extends MessageMonitorConfig {
+    type: 'redis';
+    url: string;
+    ttl?: number;
+}
+export interface InMemoryMessageMonitorConfig extends MessageMonitorConfig {
+    type: 'in-memory';
+    messagesToKeep: number;
+}
+export interface DisabledMessageMonitorConfig extends MessageMonitorConfig {
+    type: 'disabled';
+}
+export interface TopicRegex {
+    pattern: string;
+    flags: string;
+}
+export interface FileRegex {
+    pattern: string;
+    flags: string;
+}
+export declare function isTopicRegex(obj: TopicRegex | string): obj is TopicRegex;
+export interface CubeConfigHeaders {
+    Authorization: string;
+}
+export interface CubeConfig {
+    auth0_client_secret?: string;
+    auth0_client_id?: string;
+    auth0_audience?: string;
+    auth0_token_url?: string;
+    authorization?: string;
+    apiUrl: string;
+    headers?: CubeConfigHeaders;
+    redis_url?: string;
+}
+export declare function isCubeConfigType(obj: CubeConfig): obj is CubeConfig;
+export interface AwsKafkaSaslConfig {
+    region?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    mechanism: 'aws';
+}
+export declare function isAwsKafkaSalsConfig(obj: AwsKafkaSaslConfig): obj is AwsKafkaSaslConfig;
+export interface DatabaseConfig {
+    uri: string;
+    ssl?: boolean;
+    tenantPassword?: string;
+}
+export interface ActionConfig {
+    timeSensitive: boolean;
+}
+export interface KafkaBrokerConfig {
+    groupId: string;
+    clientId: string;
+    brokers: Array<string>;
+    consumerTopics?: Array<string | TopicRegex>;
+    sasl?: AwsKafkaSaslConfig;
+    intervalCheckForNewTopics?: number;
+    disableLogs?: boolean;
+    autoCommitThreshold?: number;
+    autoCommitInterval?: number;
+    partitionsConsumedConcurrently?: number;
+    messageMonitor?: RedisMessageMonitorConfig | InMemoryMessageMonitorConfig | DisabledMessageMonitorConfig;
+    useConfluentLibrary?: boolean;
+    newConsumerProtocol?: boolean;
+}
+export declare function isKafkaBrokerConfigType(obj: KafkaBrokerConfig): obj is KafkaBrokerConfig;
+export declare function isActionConfigType(obj: any): obj is ActionConfig;
+export declare function isDatabaseConfigType(obj: DatabaseConfig): obj is DatabaseConfig;
+export interface BaseConnectorConfig {
+    processIdentifier: string;
+    tenantIdentifier: string;
+    datasourceIdentifier: string;
+    kafka: KafkaBrokerConfig;
+    action?: ActionConfig;
+}
+export declare function isBaseConnectorConfigType(obj: BaseConnectorConfig): obj is BaseConnectorConfig;
+export interface AiAgentConfig {
+    openai: {
+        apiKey: string;
+        model: 'gpt-3.5-turbo' | 'gpt-4' | 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini' | 'o1-preview' | 'o1-mini';
+        temperature?: number;
+        timeout?: number;
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transai/connector-runner-file",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "publishConfig": {
     "access": "public"
   },

package/.eslintrc.json

@@ -1,18 +0,0 @@
-{
-  "extends": ["../../.eslintrc.json"],
-  "ignorePatterns": ["!**/*"],
-  "overrides": [
-    {
-      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
-      "rules": {}
-    },
-    {
-      "files": ["*.ts", "*.tsx"],
-      "rules": {}
-    },
-    {
-      "files": ["*.js", "*.jsx"],
-      "rules": {}
-    }
-  ]
-}

package/jest.config.ts

@@ -1,10 +0,0 @@
-export default {
-  displayName: 'connector-runner-file',
-  preset: '../../jest.preset.js',
-  testEnvironment: 'node',
-  transform: {
-    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
-  },
-  moduleFileExtensions: ['ts', 'js', 'html'],
-  coverageDirectory: '../../coverage/libs/connector-runner-file',
-};

package/project.json

@@ -1,33 +0,0 @@
-{
-  "name": "connector-runner-file",
-  "$schema": "../../node_modules/nx/schemas/project-schema.json",
-  "sourceRoot": "libs/connector-runner-file/src",
-  "projectType": "library",
-  "tags": [
-    "connector:runner",
-    "connector:runner-sdk",
-    "connector:file",
-    "connector:source",
-    "connector:sink",
-    "provider:@transai"
-  ],
-  "targets": {
-    "build": {
-      "executor": "@transai/tools:connector",
-      "outputs": ["{options.outputPath}"],
-      "options": {
-        "external": ["cpu-features", "ssh2"]
-      }
-    },
-    "lint": {
-      "executor": "@nx/eslint:lint"
-    },
-    "test": {
-      "executor": "@nx/jest:jest",
-      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
-      "options": {
-        "jestConfig": "libs/connector-runner-file/jest.config.ts"
-      }
-    }
-  }
-}

package/src/lib/actions-handler.spec.ts

@@ -1,121 +0,0 @@
-import {
-  ConnectorSDKInterface,
-  FilesSDKInterface,
-} from '@transai/connector-runtime-sdk';
-import { ActionInterface, XodJobType } from '@xip-online-data/types';
-
-import { ActionsHandler } from './actions-handler';
-import { FileConfig } from './types';
-
-describe('FileActionsHandler', () => {
-  let handler: ActionsHandler;
-
-  const sdkMock = {
-    logger: {
-      warn: jest.fn(),
-      info: jest.fn(),
-      debug: jest.fn(),
-      error: jest.fn(),
-      verbose: jest.fn(),
-    },
-    receiver: {
-      responses: {
-        created: jest.fn().mockReturnValue(
-          jest.fn().mockReturnValue({
-            statusCode: 201,
-          }),
-        ),
-        badRequest: jest.fn().mockReturnValue(
-          jest.fn().mockReturnValue({
-            statusCode: 400,
-          }),
-        ),
-        internalServerError: jest.fn().mockReturnValue(
-          jest.fn().mockReturnValue({
-            statusCode: 500,
-          }),
-        ),
-      },
-      emitEventType: jest.fn().mockReturnValue(
-        jest.fn().mockReturnValue({
-          statusCode: 201,
-        }),
-      ),
-    },
-  } as unknown as ConnectorSDKInterface<FileConfig>;
-  let fileHandlerMock: FilesSDKInterface;
-
-  beforeEach(() => {
-    fileHandlerMock = {
-      write: jest.fn(),
-    } as unknown as FilesSDKInterface;
-
-    handler = new ActionsHandler(sdkMock, fileHandlerMock);
-  });
-
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
-  it('should be defined', () => {
-    expect(handler).toBeDefined();
-  });
-
-  describe('JOB actions', () => {
-    const action = {
-      identifier: 'action-1',
-      version: '2.0',
-      config: {
-        parsedTemplates: {
-          filename: jest.fn().mockReturnValue('test-file.txt'),
-          contents: jest.fn().mockReturnValue('file contents here'),
-        },
-      },
-    } as unknown as ActionInterface;
-
-    const jobMessage = {
-      type: 'JOB',
-      actionIdentifier: 'action-1',
-      actionVersion: 'latest',
-      payload: { some: 'data' },
-    } as unknown as XodJobType;
-
-    it('should write file on action', async () => {
-      const response = await handler.callbackFunctionChain(jobMessage, action);
-
-      expect(fileHandlerMock.write).toHaveBeenCalledWith(
-        'test-file.txt',
-        'file contents here',
-      );
-      expect(response.statusCode).toBe(201);
-    });
-
-    it('should handle errors during file write', async () => {
-      fileHandlerMock.write = jest
-        .fn()
-        .mockRejectedValue(new Error('Write error'));
-
-      const response = await handler.callbackFunctionChain(jobMessage, action);
-
-      expect(response.statusCode).toBe(500);
-      expect(
-        sdkMock.receiver.responses.internalServerError,
-      ).toHaveBeenCalledWith('Write error');
-    });
-
-    it('should handle test run without writing file', async () => {
-      const testRunMessage = {
-        ...jobMessage,
-        testRun: true,
-      } as XodJobType;
-
-      const response = await handler.callbackFunctionChain(
-        testRunMessage,
-        action,
-      );
-
-      expect(fileHandlerMock.write).not.toHaveBeenCalled();
-      expect(response.statusCode).toBe(201);
-    });
-  });
-});

package/src/lib/actions-handler.ts

@@ -1,82 +0,0 @@
-import {
-  CompileDelegate,
-  ConnectorSDKInterface,
-  FilesSDKInterface,
-} from '@transai/connector-runtime-sdk';
-import {
-  ActionInterface,
-  KafkaCallbackResponse,
-  XodJobType,
-} from '@xip-online-data/types';
-
-export class ActionsHandler {
-  readonly #connectorSDK: ConnectorSDKInterface;
-
-  readonly #fileHandler: FilesSDKInterface;
-
-  constructor(
-    connectorSDK: ConnectorSDKInterface,
-    fileHandler: FilesSDKInterface,
-  ) {
-    this.#connectorSDK = connectorSDK;
-    this.#fileHandler = fileHandler;
-  }
-
-  get callbackFunctionChain(): (
-    message: XodJobType,
-    action: ActionInterface,
-  ) => Promise<KafkaCallbackResponse> {
-    return this.#jobCallbackFunction(
-      this.#connectorSDK.receiver.emitEventType(
-        this.#connectorSDK.receiver.responses.created(),
-      ),
-    );
-  }
-
-  #jobCallbackFunction(
-    callbackFunction: (message: XodJobType) => Promise<KafkaCallbackResponse>,
-  ) {
-    return async (
-      message: XodJobType,
-      action: ActionInterface,
-    ): Promise<KafkaCallbackResponse> => {
-      this.#connectorSDK.logger.debug(
-        `Apply templates on payload: ${JSON.stringify(message.payload)}, action: ${JSON.stringify(action)}`,
-      );
-
-      const handleBars = action.config['parsedTemplates'] as {
-        filename: CompileDelegate;
-        contents: CompileDelegate;
-      };
-
-      try {
-        const parsedFilename = handleBars
-          .filename({
-            inputs: message.payload,
-          })
-          .trim();
-
-        const parsedContent = handleBars
-          .contents({
-            inputs: message.payload,
-          })
-          .trim();
-
-        if (message.testRun) {
-          this.#connectorSDK.logger.debug(
-            `Test run for ${message.eventId} with parsedContent ${parsedContent}, parsedFilename ${parsedFilename}`,
-          );
-          return callbackFunction(message);
-        }
-
-        await this.#fileHandler.write(parsedFilename, parsedContent);
-      } catch (error) {
-        return this.#connectorSDK.receiver.responses.internalServerError(
-          error instanceof Error ? error.message : 'Unknown error occurred',
-        )(message);
-      }
-
-      return callbackFunction(message);
-    };
-  }
-}

package/src/lib/connector-runner-file.spec.ts

@@ -1,73 +0,0 @@
-import {
-  ConnectorSDKInterface,
-  FilesSDKInterface,
-} from '@transai/connector-runtime-sdk';
-import { ConnectorInterface } from '@xip-online-data/types';
-
-import { ConnectorRunnerFile } from './connector-runner-file';
-import { Processor } from './processor';
-import { FileConfig } from './types';
-
-jest.mock('./processor');
-
-describe('ConnectorRunnerFile', () => {
-  let runner: ConnectorRunnerFile;
-
-  let sdkMock: ConnectorSDKInterface<FileConfig>;
-  const processorMock = {} as unknown as Processor;
-
-  beforeEach(() => {
-    (Processor as unknown as jest.Mock).mockImplementation(() => processorMock);
-
-    sdkMock = {
-      logger: {
-        warn: jest.fn(),
-        info: jest.fn(),
-        debug: jest.fn(),
-        error: jest.fn(),
-      },
-      config: {
-        dsn: 'dummy://path/to/files',
-        fileSelectors: [{ selector: 'some-file-.+' }],
-      },
-      receiver: {
-        emitEventType: jest.fn(),
-        responses: {
-          created: jest.fn(),
-        },
-        registerCallback: jest.fn(),
-      },
-      processing: {
-        registerInterval: jest.fn(),
-      },
-      files: jest.fn().mockReturnValue({
-        pathAsDsn: (path: string) => `dummy:${path}`,
-      }),
-    } as unknown as ConnectorSDKInterface<FileConfig>;
-
-    runner = new ConnectorRunnerFile({} as ConnectorInterface, sdkMock);
-  });
-
-  it('should be defined', () => {
-    expect(runner).toBeDefined();
-  });
-
-  it('should initialize file selector processes', async () => {
-    (Processor as unknown as jest.Mock).mockImplementation(
-      (sdk, fileSelector, fileHandler: FilesSDKInterface) => {
-        expect(sdk).toBe(sdkMock);
-        expect(fileSelector).toEqual({ selector: 'some-file-.+' });
-        expect(fileHandler.pathAsDsn('/')).toEqual('dummy:/');
-
-        return processorMock;
-      },
-    );
-
-    await runner.init();
-
-    expect(sdkMock.processing.registerInterval).toHaveBeenCalledWith(
-      60,
-      processorMock,
-    );
-  });
-});

package/src/lib/connector-runner-file.ts

@@ -1,50 +0,0 @@
-import {
-  ConnectorRuntimeSDK,
-  ConnectorSDKInterface,
-  FilesSDKInterface,
-} from '@transai/connector-runtime-sdk';
-import { ConnectorInterface } from '@xip-online-data/types';
-
-import { ActionsHandler } from './actions-handler';
-import { Processor } from './processor';
-import { FileConfig } from './types';
-
-export class ConnectorRunnerFile extends ConnectorRuntimeSDK<FileConfig> {
-  readonly #fileHandler: FilesSDKInterface;
-
-  constructor(
-    connector: ConnectorInterface,
-    connectorSDK: ConnectorSDKInterface,
-  ) {
-    super(connector, connectorSDK);
-
-    const { config } = this.connectorSDK;
-    this.#fileHandler = this.connectorSDK.files(config.dsn);
-
-    const actionsHandler = new ActionsHandler(
-      this.connectorSDK,
-      this.#fileHandler,
-    );
-
-    this.callbackFunction = actionsHandler.callbackFunctionChain;
-  }
-
-  override init = async (): Promise<void> => {
-    const { config } = this.connectorSDK;
-
-    await Promise.all(
-      (config.fileSelectors ?? []).map(async (fileSelector) => {
-        const processor = new Processor(
-          this.connectorSDK,
-          fileSelector,
-          this.#fileHandler,
-        );
-
-        await this.connectorSDK.processing.registerInterval(
-          fileSelector.intervalSeconds ?? Processor.DEFAULT_INTERVAL_SECONDS,
-          processor,
-        );
-      }),
-    );
-  };
-}

package/src/lib/processor.spec.ts

@@ -1,119 +0,0 @@
-import {
-  ConnectorSDKInterface,
-  FilesSDKInterface,
-} from '@transai/connector-runtime-sdk';
-// eslint-disable-next-line @nx/enforce-module-boundaries
-import { FileHandler } from '@xip-online-data/file-handler';
-
-import { Processor } from './processor';
-import { FileConfig } from './types';
-
-jest.mock('@xip-online-data/file-handler');
-
-describe('FileProcessor', () => {
-  let processor: Processor;
-
-  const sourceDsn = 'dummy://source/path';
-  const destinationDsn = 'dummy://destination/path';
-
-  const readFile = {
-    get: jest.fn().mockReturnValue(Buffer.from('some-string')),
-    close: jest.fn().mockReturnValue(true),
-  };
-  const parsedFile = { foo: 'bar' };
-
-  const fileHandlerMock = {
-    list: jest.fn().mockResolvedValue([
-      { type: 'FILE', name: 'test-file-001.csv' },
-      { type: 'FILE', name: 'other-file.txt' },
-      { type: 'DIRECTORY', name: 'some-dir' },
-    ]),
-    read: jest.fn().mockResolvedValue(readFile),
-    delete: jest.fn(),
-  } as unknown as FilesSDKInterface;
-  const destinationFileHandlerMock = {
-    write: jest.fn(),
-    pathAsDsn: jest.fn(),
-  } as unknown as FilesSDKInterface;
-  const fileReaderMock = {
-    handleBuffer: jest.fn().mockResolvedValue(parsedFile),
-  } as unknown as FileHandler;
-
-  const sdkMock = {
-    logger: {
-      warn: jest.fn(),
-      info: jest.fn(),
-      debug: jest.fn(),
-      error: jest.fn(),
-      verbose: jest.fn(),
-    },
-    config: {
-      dsn: sourceDsn,
-    },
-    telemetry: {
-      increment: jest.fn(),
-    },
-    sender: {
-      documents: jest.fn(),
-    },
-    files: jest.fn().mockImplementation(() => destinationFileHandlerMock),
-  } as unknown as ConnectorSDKInterface<FileConfig>;
-
-  beforeEach(() => {
-    (FileHandler as unknown as jest.Mock).mockImplementation(
-      () => fileReaderMock,
-    );
-
-    processor = new Processor(
-      sdkMock,
-      {
-        selector: 'test-file-.+\\.csv',
-        delimiter: ',',
-        action: 'move',
-        destinationDsn,
-      },
-      fileHandlerMock,
-    );
-  });
-
-  it('should be defined', () => {
-    expect(processor).toBeDefined();
-  });
-
-  it('should give the correct name', () => {
-    expect(processor.name).toBe('file-processor-test-file-.+\\.csv');
-  });
-
-  it('should process files correctly', async () => {
-    await processor.onRun();
-
-    expect(fileHandlerMock.read).toHaveBeenCalledWith('/test-file-001.csv');
-    expect(fileReaderMock.handleBuffer).toHaveBeenCalledWith(
-      '/test-file-001.csv',
-      readFile.get(),
-      [],
-      undefined,
-    );
-    expect(sdkMock.sender.documents).toHaveBeenCalledWith(
-      [
-        {
-          foo: 'bar',
-          _filename: 'test-file-001.csv',
-        },
-      ],
-      {
-        keyField: '_filename',
-        collection: 'file_test-file-.+\\.csv',
-      },
-    );
-    expect(destinationFileHandlerMock.write).toHaveBeenCalledWith(
-      'test-file-001.csv',
-      readFile,
-    );
-    expect(fileHandlerMock.delete).toHaveBeenCalledWith('/test-file-001.csv');
-    expect(sdkMock.telemetry.increment).toHaveBeenCalledWith(
-      'files.processed.count',
-      1,
-    );
-  });
-});