@transai/connector-runtime-sdk 0.1.0

package/.eslintrc.json

@@ -0,0 +1,30 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.json"],
+      "parser": "jsonc-eslint-parser",
+      "rules": {
+        "@nx/dependency-checks": [
+          "error",
+          {
+            "ignoredFiles": ["{projectRoot}/eslint.config.{js,cjs,mjs}"]
+          }
+        ]
+      }
+    }
+  ]
+}

package/CHANGELOG.md

@@ -0,0 +1,25 @@
+## 0.1.0 (2025-11-19)
+
+### 🚀 Features
+
+- fix connector packages publish ([517d2fd2](https://github.com/xip-online-applications/xod-core/commit/517d2fd2))
+- **XODO-1133:** added file connector that uses the SDK ([#875](https://github.com/xip-online-applications/xod-core/pull/875))
+- **XODO-1134:** added generic API connector ([#883](https://github.com/xip-online-applications/xod-core/pull/883))
+- **XODO-1108:** added actions for MKG tables ([#877](https://github.com/xip-online-applications/xod-core/pull/877))
+- **XODO-1134:** added full http-client wrapper ([#878](https://github.com/xip-online-applications/xod-core/pull/878))
+- **XODO-1125:** use delta link in Office365 connector ([#865](https://github.com/xip-online-applications/xod-core/pull/865))
+- enhance callback handling with action configuration and job mes… ([#814](https://github.com/xip-online-applications/xod-core/pull/814))
+- **XODO-1029 XODO-1074:** added office365 mail client connector ([#805](https://github.com/xip-online-applications/xod-core/pull/805))
+- enhance callback handling with action configuration and job message validation ([225b8cef](https://github.com/xip-online-applications/xod-core/commit/225b8cef))
+- refactor output parser generation and streamline ConnectorRuntimeSDK initialization ([a72e7d83](https://github.com/xip-online-applications/xod-core/commit/a72e7d83))
+
+### 🩹 Fixes
+
+- handle failure more gracefully from OPC UA ([#856](https://github.com/xip-online-applications/xod-core/pull/856))
+
+### ❤️ Thank You
+
+- Copilot @Copilot
+- frankdikker
+- Rene Heijdens @H31nz3l
+- Youri Lefers @yourilefers

package/README.md

@@ -0,0 +1,11 @@
+# connector-runtime-sdk
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build connector-runtime-sdk` to build the library.
+
+## Running unit tests
+
+Run `nx test connector-runtime-sdk` to execute the unit tests via [Jest](https://jestjs.io).

package/jest.config.ts

@@ -0,0 +1,10 @@
+export default {
+  displayName: 'connector-runtime-sdk',
+  preset: '../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../coverage/libs/connector-runtime-sdk',
+};

package/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@transai/connector-runtime-sdk",
+  "version": "0.1.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "LGPL-3.0-or-later",
+  "author": {
+    "name": "transAI",
+    "email": "samen@transai.com",
+    "url": "https://transai.com"
+  },
+  "type": "commonjs",
+  "main": "./index.cjs",
+  "typings": "./index.d.ts"
+}

package/project.json

@@ -0,0 +1,37 @@
+{
+  "name": "@transai/connector-runtime-sdk",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/connector-runtime-sdk/src",
+  "projectType": "library",
+  "tags": ["connector:runtime", "connector:runtime-sdk-ready"],
+  "targets": {
+    "build": {
+      "executor": "@nx/esbuild:esbuild",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "platform": "node",
+        "outputPath": "dist/libs/connector-runtime-sdk",
+        "format": ["cjs", "esm"],
+        "main": "libs/connector-runtime-sdk/src/index.ts",
+        "tsConfig": "libs/connector-runtime-sdk/tsconfig.lib.json",
+        "bundle": true,
+        "deleteOutputPath": true,
+        "generatePackageJson": true,
+        "sourcemap": true,
+        "minify": true,
+        "metafile": true,
+        "thirdParty": true
+      }
+    },
+    "lint": {
+      "executor": "@nx/eslint:lint"
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "libs/connector-runtime-sdk/jest.config.ts"
+      }
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './lib/connector-runtime';
+export * from './lib/connector-runtime.interface';
+export * from './lib/sdk';

package/src/lib/connector-runtime.interface.ts

@@ -0,0 +1,7 @@
+export interface ConnectorRuntimeInterface {
+  init(): Promise<void>;
+
+  start(): Promise<void>;
+
+  stop(): Promise<void>;
+}

package/src/lib/connector-runtime.spec.ts

@@ -0,0 +1,115 @@
+import {
+  ConfiguredConnectorTypes,
+  ConnectorInterface,
+  KafkaCallbackResponse,
+  XodJobType,
+} from '@xip-online-data/types';
+
+import { ConnectorRuntimeSDK } from './connector-runtime';
+import {
+  CompileDelegate,
+  ConnectorSDKInterface,
+  LoggerSDKInterface,
+} from './sdk';
+
+class Implementation extends ConnectorRuntimeSDK {
+  override init = async (): Promise<void> => {
+    this.callbackFunction = (
+      message: XodJobType,
+    ): Promise<KafkaCallbackResponse> => {
+      throw new Error(`Method not implemented, called with ${message.type}`);
+    };
+  };
+}
+
+describe('ConnectorRuntimeSDK', () => {
+  let implementation: Implementation;
+  let connector: ConnectorInterface;
+  let connectorSDK: ConnectorSDKInterface;
+
+  beforeEach(() => {
+    connector = {
+      identifier: 'test-connector',
+      connectorType: ConfiguredConnectorTypes.DUMMY_NODE,
+      tenantIdentifier: 'test-tenant',
+      name: 'Test Connector',
+      location: 'us-east-1',
+      enabled: true,
+      config: {},
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      actions: [
+        {
+          identifier: 'test-action',
+          version: '1.0.0',
+          tenantIdentifier: 'test-tenant',
+          connectorIdentifier: 'test-connector',
+          name: 'Test Action',
+          config: {
+            templates: {
+              testTemplate: 'Hello, {{name}',
+            },
+          },
+          inputParameters: [],
+          outputParameters: {},
+          createdAt: new Date(),
+        },
+      ],
+    };
+    connectorSDK = {
+      logger: {
+        info: jest.fn(),
+        warn: jest.fn(),
+        error: jest.fn(),
+        debug: jest.fn(),
+        verbose: jest.fn(),
+      } as LoggerSDKInterface,
+      templating: { compile: jest.fn().mockReturnValue({} as CompileDelegate) },
+      receiver: {
+        registerCallback: jest.fn(),
+      },
+    } as unknown as ConnectorSDKInterface;
+
+    implementation = new Implementation(connector, connectorSDK);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should be defined', () => {
+    expect(implementation).toBeDefined();
+  });
+
+  describe('actions compilation', () => {
+    it('should have compiled templates for actions', () => {
+      expect(connectorSDK.templating.compile).toHaveBeenCalledWith(
+        'Hello, {{name}',
+      );
+    });
+
+    it('should have parsed templates in action config', () => {
+      expect(
+        connector.actions?.[0].config['parsedTemplates']?.['testTemplate'],
+      ).toBeDefined();
+    });
+  });
+
+  describe('callback registration', () => {
+    it('should have registered the callback function on init', async () => {
+      await implementation.init();
+
+      expect(connectorSDK.receiver.registerCallback).toHaveBeenCalled();
+    });
+  });
+
+  describe('lifecycle', () => {
+    it('should start without error', async () => {
+      await expect(implementation.start()).resolves.toBeUndefined();
+    });
+
+    it('should stop without error', async () => {
+      await expect(implementation.stop()).resolves.toBeUndefined();
+    });
+  });
+});

package/src/lib/connector-runtime.ts

@@ -0,0 +1,119 @@
+import * as process from 'node:process';
+
+import {
+  KafkaCallbackResponse,
+  BaseConnectorConfig,
+  ActionInterface,
+  ConnectorInterface,
+  XodJobType,
+} from '@xip-online-data/types';
+
+import { ConnectorRuntimeInterface } from './connector-runtime.interface';
+import { CompileDelegate, ConnectorSDKInterface } from './sdk';
+
+export interface IpcMessage {
+  cmd: string;
+  message: string;
+}
+
+export abstract class ConnectorRuntimeSDK<
+  T extends BaseConnectorConfig = BaseConnectorConfig,
+> implements ConnectorRuntimeInterface
+{
+  readonly #IPC_CHANNEL = 'connector-runtime';
+
+  readonly #connectorSDK: ConnectorSDKInterface<T>;
+
+  constructor(
+    connector: ConnectorInterface,
+    connectorSDK: ConnectorSDKInterface,
+  ) {
+    this.#connectorSDK = connectorSDK as ConnectorSDKInterface<T>;
+
+    if (process.on) {
+      process.on('message', (message: IpcMessage) => {
+        if (message.cmd === this.#IPC_CHANNEL) {
+          connectorSDK.logger.verbose(
+            `${process.pid} Received message from parent process:`,
+            message,
+          );
+        }
+      });
+    } else {
+      connectorSDK.logger.warn(
+        'IPC channel is not available. process.on is undefined.',
+      );
+    }
+
+    if (!process.send) {
+      // Check if process.send is available
+      connectorSDK.logger.warn(
+        'IPC channel is not available. process.send is undefined.',
+      );
+    }
+
+    // eslint-disable-next-line no-restricted-syntax
+    connector.actions?.forEach((action: ActionInterface) => {
+      if (action.config['templates'] === undefined) {
+        return;
+      }
+
+      const containers: {
+        [key: string]: CompileDelegate;
+      } = {};
+
+      Object.entries(action.config['templates']).forEach(([key, value]) => {
+        try {
+          // eslint-disable-next-line security/detect-object-injection
+          containers[key] = connectorSDK.templating.compile(value);
+        } catch (error: unknown) {
+          connectorSDK.logger.error(
+            `Error compiling template ${key} for action ${action.identifier}`,
+            error,
+          );
+        }
+      });
+
+      // eslint-disable-next-line no-param-reassign
+      action.config['parsedTemplates'] = containers;
+    });
+  }
+
+  public init: () => Promise<void> = () => Promise.resolve();
+
+  public start: () => Promise<void> = () => Promise.resolve();
+
+  public stop: () => Promise<void> = () => Promise.resolve();
+
+  set callbackFunction(
+    callback: (
+      message: XodJobType,
+      action: ActionInterface,
+    ) => Promise<KafkaCallbackResponse>,
+  ) {
+    this.connectorSDK.receiver.registerCallback(
+      this.#enrichWithActionConfig(callback),
+    );
+  }
+
+  protected get connectorSDK(): ConnectorSDKInterface<T> {
+    return this.#connectorSDK;
+  }
+
+  #enrichWithActionConfig(
+    callbackFunction: (
+      message: XodJobType,
+      action: ActionInterface,
+    ) => Promise<KafkaCallbackResponse>,
+  ): (message: XodJobType) => Promise<KafkaCallbackResponse> {
+    return async (message: XodJobType) => {
+      const action = this.#connectorSDK.receiver.getActionConfig(message);
+      if (!action) {
+        return this.#connectorSDK.receiver.responses.badRequest(
+          'Action not found',
+        )(message);
+      }
+      return callbackFunction(message, action);
+    };
+  }
+}

package/src/lib/sdk/files.sdk.interface.ts

@@ -0,0 +1,29 @@
+export interface FileInfo {
+  type: 'DIRECTORY' | 'FILE';
+  name: string;
+  size: number;
+  modifyTime: Date;
+}
+
+export interface FileHandleInterface {
+  get(): Buffer;
+
+  close(): boolean;
+}
+
+export interface FilesSDKInterface {
+  list(path: string): Promise<Array<FileInfo>>;
+
+  read(filePath: string): Promise<FileHandleInterface>;
+
+  write(
+    filePath: string,
+    data: FileHandleInterface | Buffer | string,
+  ): Promise<boolean>;
+
+  delete(path: string): Promise<boolean>;
+
+  exists(path: string): Promise<boolean>;
+
+  pathAsDsn(path: string): string;
+}

package/src/lib/sdk/http-client.interface.ts

@@ -0,0 +1,110 @@
+export interface HttpConfig {
+  baseUrl?: string;
+  contentType?: string;
+  timeout?: number;
+}
+
+export type HttpRequestOptions<D = string | object> = {
+  headers?: Record<string, string>;
+  params?: Record<string, string | number | boolean>;
+  data?: D;
+  withCredentials?: boolean;
+};
+
+export type HttpRequestOptionsFormatter<D = string | object> = (
+  requestOptions: HttpRequestOptions<D>,
+  method: HttpMethod,
+  url: string,
+) => Promise<HttpRequestOptions<D>> | HttpRequestOptions<D>;
+
+export type HttpMethod =
+  | 'GET'
+  | 'POST'
+  | 'PUT'
+  | 'DELETE'
+  | 'PATCH'
+  | 'HEAD'
+  | 'OPTIONS';
+
+export type HttpResponseHeader =
+  | string
+  | Array<string>
+  | number
+  | boolean
+  | null;
+
+export type HttpResponse<D = unknown> = {
+  status: number;
+  data: D;
+  headers?: Record<string, HttpResponseHeader>;
+};
+
+export type HttpError = {
+  status: number;
+  error?: string;
+};
+
+interface PromiseWithReason<T = unknown, R = unknown> extends Promise<T> {
+  then<TResult1 = T, TResult2 = never>(
+    onfulfilled?:
+      | ((value: T) => TResult1 | PromiseLike<TResult1>)
+      | undefined
+      | null,
+    onrejected?:
+      | ((reason: R) => TResult2 | PromiseLike<TResult2>)
+      | undefined
+      | null,
+  ): Promise<TResult1 | TResult2>;
+
+  catch<TResult = never>(
+    onrejected?:
+      | ((reason: R) => TResult | PromiseLike<TResult>)
+      | undefined
+      | null,
+  ): Promise<T | TResult>;
+}
+
+export type HttpClientPromise<T> = PromiseWithReason<
+  HttpResponse<T>,
+  HttpError
+>;
+
+export interface HttpClientSDKInterface {
+  get<R = unknown>(
+    destination: string,
+    options?: HttpRequestOptions,
+  ): HttpClientPromise<R>;
+
+  post<R = unknown, D = string | object>(
+    destination: string,
+    data: D,
+    options?: HttpRequestOptions<D>,
+  ): HttpClientPromise<R>;
+
+  put<R = unknown, D = string | object>(
+    destination: string,
+    data: D,
+    options?: HttpRequestOptions<D>,
+  ): HttpClientPromise<R>;
+
+  patch<R = unknown, D = string | object>(
+    destination: string,
+    data: D,
+    options?: HttpRequestOptions<D>,
+  ): HttpClientPromise<R>;
+
+  delete<R = unknown>(
+    destination: string,
+    options?: HttpRequestOptions,
+  ): HttpClientPromise<R>;
+
+  request<R = unknown, D = string | object>(
+    method: HttpMethod,
+    url: string,
+    options?: HttpRequestOptions<D>,
+  ): HttpClientPromise<R>;
+
+  setRequestOptionsFormatter<D = string | object>(
+    formatter: HttpRequestOptionsFormatter<D>,
+  ): HttpClientSDKInterface;
+}

package/src/lib/sdk/index.ts

@@ -0,0 +1,10 @@
+export * from './sdk.interface';
+export * from './http-client.interface';
+export * from './logger.sdk.interface';
+export * from './offset-store.sdk.interface';
+export * from './processing.sdk.interface';
+export * from './receiver.sdk.interface';
+export * from './sender.sdk.interface';
+export * from './telemetry.sdk.interface';
+export * from './templating.sdk.interface';
+export * from './files.sdk.interface';

package/src/lib/sdk/logger.sdk.interface.ts

@@ -0,0 +1,11 @@
+export interface LoggerSDKInterface {
+  info(...args: Array<unknown>): void;
+
+  debug(...args: Array<unknown>): void;
+
+  error(...args: Array<unknown>): void;
+
+  warn(...args: Array<unknown>): void;
+
+  verbose(...args: Array<unknown>): void;
+}

package/src/lib/sdk/offset-store.sdk.interface.ts

@@ -0,0 +1,14 @@
+export interface StoredOffset {
+  timestamp: number;
+  id: number | string;
+  isoDate?: string;
+  date?: string;
+  rawTimestamp?: number | string;
+  [key: string]: string | number | undefined;
+}
+
+export interface OffsetStoreSDKInterface {
+  getOffset(identifier: string): Promise<StoredOffset>;
+
+  setOffset(offset: StoredOffset, identifier: string): void;
+}

package/src/lib/sdk/processing.sdk.interface.ts

@@ -0,0 +1,20 @@
+export interface IntervalHandler {
+  name?: string;
+  onInit?: () => Promise<void> | void;
+  onRun: () => Promise<void> | void;
+  onStop?: () => Promise<void> | void;
+}
+
+export type RegisterIntervalOptions = {
+  immediate?: boolean;
+};
+
+export interface ProcessingSDKInterface {
+  registerInterval(
+    intervalSeconds: number,
+    handler: IntervalHandler,
+    options?: RegisterIntervalOptions,
+  ): Promise<string>;
+
+  stopInterval(name: string): Promise<void>;
+}

package/src/lib/sdk/receiver.sdk.interface.ts

@@ -0,0 +1,29 @@
+import {
+  ActionInterface,
+  KafkaCallbackResponse,
+  KafkaCallbackResponseType,
+  XodJobType,
+} from '@xip-online-data/types';
+
+export interface ReceiverSDKInterface {
+  readonly responses: {
+    readonly ok: KafkaCallbackResponseType;
+    readonly created: KafkaCallbackResponseType;
+    readonly badRequest: KafkaCallbackResponseType;
+    readonly unprocessableEntity: KafkaCallbackResponseType;
+    readonly notFound: KafkaCallbackResponseType;
+    readonly internalServerError: KafkaCallbackResponseType;
+  };
+
+  registerCallback<T extends XodJobType = XodJobType>(
+    callbackFunction: (message: T) => Promise<KafkaCallbackResponse>,
+    eventType?: string,
+    identifier?: string,
+  ): void;
+
+  getActionConfig(message: XodJobType): ActionInterface | null;
+
+  emitEventType<T extends XodJobType = XodJobType>(
+    callbackFunction: (message: T) => Promise<KafkaCallbackResponse>,
+  ): (message: T) => Promise<KafkaCallbackResponse>;
+}

package/src/lib/sdk/sdk.interface.ts

@@ -0,0 +1,42 @@
+import { BaseConnectorConfig } from '@xip-online-data/types';
+
+import { FilesSDKInterface } from './files.sdk.interface';
+import { HttpClientSDKInterface, HttpConfig } from './http-client.interface';
+import { LoggerSDKInterface } from './logger.sdk.interface';
+import { OffsetStoreSDKInterface } from './offset-store.sdk.interface';
+import { ProcessingSDKInterface } from './processing.sdk.interface';
+import { ReceiverSDKInterface } from './receiver.sdk.interface';
+import { SenderSDKInterface } from './sender.sdk.interface';
+import { TelemetrySDKInterface } from './telemetry.sdk.interface';
+import { TemplatingSDKInterface } from './templating.sdk.interface';
+
+export interface ConnectorSDKInterface<T = BaseConnectorConfig> {
+  get config(): T;
+
+  get logger(): LoggerSDKInterface;
+
+  get sender(): SenderSDKInterface;
+
+  get receiver(): ReceiverSDKInterface;
+
+  get templating(): TemplatingSDKInterface;
+
+  /**
+   * Service to schedule and manage processing tasks that need monitoring and retries.
+   */
+  get processing(): ProcessingSDKInterface;
+
+  get offsetStore(): OffsetStoreSDKInterface;
+
+  /**
+   * Ability to send telemetry data such as gauges and increments on how the connector is performing.
+   */
+  get telemetry(): TelemetrySDKInterface;
+
+  /**
+   * Get file service instance to access files using a specified DSN. If no DSN is provided, defaults to local file system tmp dir.
+   */
+  files(dsn?: string): FilesSDKInterface;
+
+  httpClient(httpConfig?: HttpConfig): HttpClientSDKInterface;
+}

package/src/lib/sdk/sender.sdk.interface.ts

@@ -0,0 +1,34 @@
+export interface Metric {
+  key: string;
+  value: number;
+}
+
+export type Metadata = { [key: string]: string };
+
+export interface Context {
+  timestamp?: Date;
+  ttl?: number;
+  extraPayload?: { [key: string]: string };
+}
+
+export type Result = boolean | { [index: number]: boolean };
+
+export interface SenderSDKInterface {
+  metrics(
+    metrics: Array<Metric>,
+    metadata?: Metadata,
+    context?: Context,
+  ): Promise<Result>;
+
+  metricsLegacy(
+    metrics: Array<never>,
+    metadata?: Metadata,
+    context?: Context,
+  ): Promise<Result>;
+
+  documents(
+    metrics: Array<object>,
+    metadata?: Metadata,
+    context?: Context,
+  ): Promise<Result>;
+}

package/src/lib/sdk/telemetry.sdk.interface.ts

@@ -0,0 +1,12 @@
+export interface TelemetrySDKInterface {
+  /**
+   * Increment a telemetry counter by 1.
+   */
+  increment(key: string): void;
+
+  /**
+   * Increment a telemetry counter by a specified value. 0 doesn't send any telemetry.
+   * A negative value will decrement the counter.
+   */
+  increment(key: string, value: number): void;
+}

package/src/lib/sdk/templating.sdk.interface.ts

@@ -0,0 +1,16 @@
+export type RuntimeOptions = {
+  partial?: boolean;
+};
+
+export type CompileDelegate<T = unknown> = {
+  (context: T, options?: RuntimeOptions): string;
+};
+
+export type CompileOptions = { strict?: boolean };
+
+export interface TemplatingSDKInterface {
+  compile<T = unknown>(
+    input: string,
+    options?: CompileOptions,
+  ): CompileDelegate<T>;
+}

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "module": "commonjs"
+  },
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"],
+  "include": ["src/**/*.ts"]
+}

package/tsconfig.spec.json

@@ -0,0 +1,14 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": [
+    "jest.config.ts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}