@transai/connector-runner-mkg 0.2.0 → 0.4.0

package/libs/connector-runner-mkg/src/lib/tables/_all.d.ts

@@ -0,0 +1,4 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLES: {
+    [identifier: string]: MkgTable;
+};

package/libs/connector-runner-mkg/src/lib/tables/artg.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_ARTG: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/arti.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_ARTI: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/base/action.d.ts

@@ -0,0 +1,21 @@
+import { MkgTableInterface } from '../../types';
+type MkgActionMethod = 'GET' | 'POST' | 'PUT' | 'DELETE';
+export interface MkgActionInterface {
+    identifier: string;
+    method: MkgActionMethod;
+    fields: Array<string>;
+    optionalFields?: Array<string>;
+    path?: (inputData: Record<string, unknown>) => string;
+}
+export declare class MkgAction implements MkgActionInterface {
+    #private;
+    readonly identifier: string;
+    readonly method: MkgActionMethod;
+    readonly fields: Array<string>;
+    readonly optionalFields: Array<string>;
+    readonly path: (inputData: Record<string, unknown>) => string;
+    constructor(params: MkgActionInterface);
+    withParentTable(table: MkgTableInterface): MkgAction;
+    fieldSet(inputData: Record<string, unknown>): Record<string, unknown>;
+}
+export {};

package/libs/connector-runner-mkg/src/lib/tables/base/table.d.ts

@@ -0,0 +1,15 @@
+import { MkgTableInterface, TableConfig } from '../../types';
+import { MkgAction } from './action';
+export declare class MkgTable implements MkgTableInterface {
+    readonly identifier: string;
+    readonly fields: Array<string>;
+    readonly immediate: boolean;
+    readonly interval: number;
+    readonly identifierField: string;
+    readonly dateField: string;
+    readonly rows: number;
+    readonly actions: Array<MkgAction>;
+    constructor(params: MkgTableInterface, actions?: Array<MkgAction>);
+    cloneFromTableConfig(tableConfig: TableConfig | true): MkgTable;
+    action(identifier: string): MkgAction | undefined;
+}

package/libs/connector-runner-mkg/src/lib/tables/bwrg.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_BWRG: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/bwrk.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_BWRK: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/clch.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_CLCH: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/debi.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_DEBI: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/magl.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_MAGL: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/magz.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_MAGZ: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/medw.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_MEDW: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/parl.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_PARL: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/plnb.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_PLNB: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/prbv.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_PRBV: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/prdh.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_PRDH: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/prdr.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_PRDR: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/prmv.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_PRMV: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/rela.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_RELA: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/rgrs.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_RGRS: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/rsrc.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_RSRC: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/rsrd.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_RSRD: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/rsrg.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_RSRG: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/stlh.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_STLH: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/stlm.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_STLM: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/stlr.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_STLR: MkgTable;

package/libs/connector-runner-mkg/src/lib/tables/vrdg.d.ts

@@ -0,0 +1,2 @@
+import { MkgTable } from './base/table';
+export declare const MKG_TABLE_VRDG: MkgTable;

package/libs/connector-runner-mkg/src/lib/types.d.ts

@@ -0,0 +1,34 @@
+import { BaseConnectorConfig } from '@xip-online-data/types';
+export declare const DEFAULT_DATASOURCE = "mkg";
+export declare const DEFAULT_INTERVAL = 3600;
+export declare const DEFAULT_ROW_KEY_FIELD = "RowKey";
+export declare const DEFAULT_DATE_FIELD = "sys_dat_wijzig";
+export declare const DEFAULT_NUM_ROWS = 250;
+export declare const SESSION_COOKIE_NAME = "JSESSIONID";
+/**
+ * Session expires in 30 minutes
+ */
+export declare const SESSION_EXPIRATION_SECONDS: number;
+export interface TableConfig {
+    fields?: Array<string>;
+}
+export interface ConnectorConfig extends BaseConnectorConfig {
+    server: 'saas*.mkg.eu' | string;
+    port?: 443 | 8443 | number;
+    path?: '/mkg' | '/mkgoefenclient' | string;
+    username: string;
+    password: string;
+    apiToken: string;
+    tables?: {
+        [key: string]: TableConfig | true;
+    };
+}
+export interface MkgTableInterface {
+    identifier: string;
+    fields: Array<string>;
+    immediate?: boolean;
+    interval?: number;
+    identifierField?: string;
+    dateField?: string;
+    rows?: number;
+}

package/libs/connector-runtime-sdk/src/index.d.ts

@@ -0,0 +1,3 @@
+export * from './lib/connector-runtime';
+export * from './lib/connector-runtime.interface';
+export * from './lib/sdk';

package/libs/connector-runtime-sdk/src/lib/connector-runtime.d.ts

@@ -0,0 +1,16 @@
+import { KafkaCallbackResponse, BaseConnectorConfig, ActionInterface, ConnectorInterface, XodJobType } from '@xip-online-data/types';
+import { ConnectorRuntimeInterface } from './connector-runtime.interface';
+import { ConnectorSDKInterface } from './sdk';
+export interface IpcMessage {
+    cmd: string;
+    message: string;
+}
+export declare abstract class ConnectorRuntimeSDK<T extends BaseConnectorConfig = BaseConnectorConfig> implements ConnectorRuntimeInterface {
+    #private;
+    constructor(connector: ConnectorInterface, connectorSDK: ConnectorSDKInterface);
+    init: () => Promise<void>;
+    start: () => Promise<void>;
+    stop: () => Promise<void>;
+    set callbackFunction(callback: (message: XodJobType, action: ActionInterface) => Promise<KafkaCallbackResponse>);
+    protected get connectorSDK(): ConnectorSDKInterface<T>;
+}

package/libs/connector-runtime-sdk/src/lib/connector-runtime.interface.d.ts

@@ -0,0 +1,5 @@
+export interface ConnectorRuntimeInterface {
+    init(): Promise<void>;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/files.sdk.interface.d.ts

@@ -0,0 +1,18 @@
+export interface FileInfo {
+    type: 'DIRECTORY' | 'FILE';
+    name: string;
+    size: number;
+    modifyTime: Date;
+}
+export interface FileHandleInterface {
+    get(): Buffer;
+    close(): boolean;
+}
+export interface FilesSDKInterface {
+    list(path: string): Promise<Array<FileInfo>>;
+    read(filePath: string): Promise<FileHandleInterface>;
+    write(filePath: string, data: FileHandleInterface | Buffer | string): Promise<boolean>;
+    delete(path: string): Promise<boolean>;
+    exists(path: string): Promise<boolean>;
+    pathAsDsn(path: string): string;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/http-client.interface.d.ts

@@ -0,0 +1,38 @@
+export interface HttpConfig {
+    baseUrl?: string;
+    contentType?: string;
+    timeout?: number;
+}
+export type HttpRequestOptions<D = string | object> = {
+    headers?: Record<string, string>;
+    params?: Record<string, string | number | boolean>;
+    data?: D;
+    withCredentials?: boolean;
+};
+export type HttpRequestOptionsFormatter<D = string | object> = (requestOptions: HttpRequestOptions<D>, method: HttpMethod, url: string) => Promise<HttpRequestOptions<D>> | HttpRequestOptions<D>;
+export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
+export type HttpResponseHeader = string | Array<string> | number | boolean | null;
+export type HttpResponse<D = unknown> = {
+    status: number;
+    data: D;
+    headers?: Record<string, HttpResponseHeader>;
+};
+export type HttpError = {
+    status: number;
+    error?: string;
+};
+interface PromiseWithReason<T = unknown, R = unknown> extends Promise<T> {
+    then<TResult1 = T, TResult2 = never>(onfulfilled?: ((value: T) => TResult1 | PromiseLike<TResult1>) | undefined | null, onrejected?: ((reason: R) => TResult2 | PromiseLike<TResult2>) | undefined | null): Promise<TResult1 | TResult2>;
+    catch<TResult = never>(onrejected?: ((reason: R) => TResult | PromiseLike<TResult>) | undefined | null): Promise<T | TResult>;
+}
+export type HttpClientPromise<T> = PromiseWithReason<HttpResponse<T>, HttpError>;
+export interface HttpClientSDKInterface {
+    get<R = unknown>(destination: string, options?: HttpRequestOptions): HttpClientPromise<R>;
+    post<R = unknown, D = string | object>(destination: string, data: D, options?: HttpRequestOptions<D>): HttpClientPromise<R>;
+    put<R = unknown, D = string | object>(destination: string, data: D, options?: HttpRequestOptions<D>): HttpClientPromise<R>;
+    patch<R = unknown, D = string | object>(destination: string, data: D, options?: HttpRequestOptions<D>): HttpClientPromise<R>;
+    delete<R = unknown>(destination: string, options?: HttpRequestOptions): HttpClientPromise<R>;
+    request<R = unknown, D = string | object>(method: HttpMethod, url: string, options?: HttpRequestOptions<D>): HttpClientPromise<R>;
+    setRequestOptionsFormatter<D = string | object>(formatter: HttpRequestOptionsFormatter<D>): HttpClientSDKInterface;
+}
+export {};

package/libs/connector-runtime-sdk/src/lib/sdk/index.d.ts

@@ -0,0 +1,10 @@
+export * from './sdk.interface';
+export * from './http-client.interface';
+export * from './logger.sdk.interface';
+export * from './offset-store.sdk.interface';
+export * from './processing.sdk.interface';
+export * from './receiver.sdk.interface';
+export * from './sender.sdk.interface';
+export * from './telemetry.sdk.interface';
+export * from './templating.sdk.interface';
+export * from './files.sdk.interface';

package/libs/connector-runtime-sdk/src/lib/sdk/logger.sdk.interface.d.ts

@@ -0,0 +1,7 @@
+export interface LoggerSDKInterface {
+    info(...args: Array<unknown>): void;
+    debug(...args: Array<unknown>): void;
+    error(...args: Array<unknown>): void;
+    warn(...args: Array<unknown>): void;
+    verbose(...args: Array<unknown>): void;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/offset-store.sdk.interface.d.ts

@@ -0,0 +1,12 @@
+export interface StoredOffset {
+    timestamp: number;
+    id: number | string;
+    isoDate?: string;
+    date?: string;
+    rawTimestamp?: number | string;
+    [key: string]: string | number | undefined;
+}
+export interface OffsetStoreSDKInterface {
+    getOffset(identifier: string): Promise<StoredOffset>;
+    setOffset(offset: StoredOffset, identifier: string): void;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/processing.sdk.interface.d.ts

@@ -0,0 +1,13 @@
+export interface IntervalHandler {
+    name?: string;
+    onInit?: () => Promise<void> | void;
+    onRun: () => Promise<void> | void;
+    onStop?: () => Promise<void> | void;
+}
+export type RegisterIntervalOptions = {
+    immediate?: boolean;
+};
+export interface ProcessingSDKInterface {
+    registerInterval(intervalSeconds: number, handler: IntervalHandler, options?: RegisterIntervalOptions): Promise<string>;
+    stopInterval(name: string): Promise<void>;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/receiver.sdk.interface.d.ts

@@ -0,0 +1,14 @@
+import { ActionInterface, KafkaCallbackResponse, KafkaCallbackResponseType, XodJobType } from '@xip-online-data/types';
+export interface ReceiverSDKInterface {
+    readonly responses: {
+        readonly ok: KafkaCallbackResponseType;
+        readonly created: KafkaCallbackResponseType;
+        readonly badRequest: KafkaCallbackResponseType;
+        readonly unprocessableEntity: KafkaCallbackResponseType;
+        readonly notFound: KafkaCallbackResponseType;
+        readonly internalServerError: KafkaCallbackResponseType;
+    };
+    registerCallback<T extends XodJobType = XodJobType>(callbackFunction: (message: T) => Promise<KafkaCallbackResponse>, eventType?: string, identifier?: string): void;
+    getActionConfig(message: XodJobType): ActionInterface | null;
+    emitEventType<T extends XodJobType = XodJobType>(callbackFunction: (message: T) => Promise<KafkaCallbackResponse>): (message: T) => Promise<KafkaCallbackResponse>;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/sdk.interface.d.ts

@@ -0,0 +1,31 @@
+import { BaseConnectorConfig } from '@xip-online-data/types';
+import { FilesSDKInterface } from './files.sdk.interface';
+import { HttpClientSDKInterface, HttpConfig } from './http-client.interface';
+import { LoggerSDKInterface } from './logger.sdk.interface';
+import { OffsetStoreSDKInterface } from './offset-store.sdk.interface';
+import { ProcessingSDKInterface } from './processing.sdk.interface';
+import { ReceiverSDKInterface } from './receiver.sdk.interface';
+import { SenderSDKInterface } from './sender.sdk.interface';
+import { TelemetrySDKInterface } from './telemetry.sdk.interface';
+import { TemplatingSDKInterface } from './templating.sdk.interface';
+export interface ConnectorSDKInterface<T = BaseConnectorConfig> {
+    get config(): T;
+    get logger(): LoggerSDKInterface;
+    get sender(): SenderSDKInterface;
+    get receiver(): ReceiverSDKInterface;
+    get templating(): TemplatingSDKInterface;
+    /**
+     * Service to schedule and manage processing tasks that need monitoring and retries.
+     */
+    get processing(): ProcessingSDKInterface;
+    get offsetStore(): OffsetStoreSDKInterface;
+    /**
+     * Ability to send telemetry data such as gauges and increments on how the connector is performing.
+     */
+    get telemetry(): TelemetrySDKInterface;
+    /**
+     * Get file service instance to access files using a specified DSN. If no DSN is provided, defaults to local file system tmp dir.
+     */
+    files(dsn?: string): FilesSDKInterface;
+    httpClient(httpConfig?: HttpConfig): HttpClientSDKInterface;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/sender.sdk.interface.d.ts

@@ -0,0 +1,22 @@
+export interface Metric {
+    key: string;
+    value: number;
+}
+export type Metadata = {
+    [key: string]: string;
+};
+export interface Context {
+    timestamp?: Date;
+    ttl?: number;
+    extraPayload?: {
+        [key: string]: string;
+    };
+}
+export type Result = boolean | {
+    [index: number]: boolean;
+};
+export interface SenderSDKInterface {
+    metrics(metrics: Array<Metric>, metadata?: Metadata, context?: Context): Promise<Result>;
+    metricsLegacy(metrics: Array<never>, metadata?: Metadata, context?: Context): Promise<Result>;
+    documents(metrics: Array<object>, metadata?: Metadata, context?: Context): Promise<Result>;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/telemetry.sdk.interface.d.ts

@@ -0,0 +1,15 @@
+export interface TelemetrySDKInterface {
+    /**
+     * Increment a telemetry counter by 1.
+     */
+    increment(key: string): void;
+    /**
+     * Increment a telemetry counter by a specified value. 0 doesn't send any telemetry.
+     * A negative value will decrement the counter.
+     */
+    increment(key: string, value: number): void;
+    /**
+     * Send a gauge telemetry metric.
+     */
+    gauge(key: string, value: number): void;
+}

package/libs/connector-runtime-sdk/src/lib/sdk/templating.sdk.interface.d.ts

@@ -0,0 +1,12 @@
+export type RuntimeOptions = {
+    partial?: boolean;
+};
+export type CompileDelegate<T = unknown> = {
+    (context: T, options?: RuntimeOptions): string;
+};
+export type CompileOptions = {
+    strict?: boolean;
+};
+export interface TemplatingSDKInterface {
+    compile<T = unknown>(input: string, options?: CompileOptions): CompileDelegate<T>;
+}

package/libs/types/src/index.d.ts

@@ -0,0 +1,7 @@
+export * from './lib/types';
+export * from './lib/response.types';
+export * from './lib/message.types';
+export * from './lib/cube-query-config.types';
+export * from './lib/file-action.types';
+export * from './lib/management-api';
+export * from './lib/http-status-codes.enum';

package/libs/types/src/lib/cube-query-config.types.d.ts

@@ -0,0 +1,20 @@
+import { Query } from '@cubejs-client/core';
+export interface CubeQueryConfig {
+    tenantId: string;
+    query: Query;
+    nullable?: boolean;
+    subqueries: Array<CubeSubQueryConfig>;
+}
+export interface CubeSubQueryConfig {
+    tenantId?: string;
+    query: Query;
+    nullable?: boolean;
+    field: string;
+    joinConditions: Array<CubeJoinCondition>;
+}
+export interface CubeJoinCondition {
+    selectField: string;
+    operator: string;
+    valueField: Array<string>;
+    valueMap?: Array<any>;
+}

package/libs/types/src/lib/file-action.types.d.ts

@@ -0,0 +1,5 @@
+export declare enum FileActionType {
+    ACTION_MOVE = "move",
+    ACTION_DELETE = "delete",
+    ACTION_NOTHING = "nothing"
+}

package/libs/types/src/lib/http-status-codes.enum.d.ts

@@ -0,0 +1,64 @@
+export declare enum HttpStatusCode {
+    Continue = 100,
+    SwitchingProtocols = 101,
+    Processing = 102,
+    OK = 200,
+    Created = 201,
+    Accepted = 202,
+    NonAuthoritativeInformation = 203,
+    NoContent = 204,
+    ResetContent = 205,
+    PartialContent = 206,
+    MultiStatus = 207,
+    AlreadyReported = 208,
+    IMUsed = 226,
+    MultipleChoices = 300,
+    MovedPermanently = 301,
+    Found = 302,
+    SeeOther = 303,
+    NotModified = 304,
+    UseProxy = 305,
+    TemporaryRedirect = 307,
+    PermanentRedirect = 308,
+    BadRequest = 400,
+    Unauthorized = 401,
+    PaymentRequired = 402,
+    Forbidden = 403,
+    NotFound = 404,
+    MethodNotAllowed = 405,
+    NotAcceptable = 406,
+    ProxyAuthenticationRequired = 407,
+    RequestTimeout = 408,
+    Conflict = 409,
+    Gone = 410,
+    LengthRequired = 411,
+    PreconditionFailed = 412,
+    PayloadTooLarge = 413,
+    URITooLong = 414,
+    UnsupportedMediaType = 415,
+    RangeNotSatisfiable = 416,
+    ExpectationFailed = 417,
+    ImATeapot = 418,
+    MisdirectedRequest = 421,
+    UnprocessableEntity = 422,
+    Locked = 423,
+    FailedDependency = 424,
+    TooEarly = 425,
+    UpgradeRequired = 426,
+    PreconditionRequired = 428,
+    TooManyRequests = 429,
+    RequestHeaderFieldsTooLarge = 431,
+    UnavailableForLegalReasons = 451,
+    InternalServerError = 500,
+    NotImplemented = 501,
+    BadGateway = 502,
+    ServiceUnavailable = 503,
+    GatewayTimeout = 504,
+    HTTPVersionNotSupported = 505,
+    VariantAlsoNegotiates = 506,
+    InsufficientStorage = 507,
+    LoopDetected = 508,
+    NotExtended = 510,
+    NetworkAuthenticationRequired = 511,
+    Aborted = 512
+}

package/libs/types/src/lib/management-api/action-definition.interface.d.ts

@@ -0,0 +1,12 @@
+import { TenantInterface } from './tenant.interface';
+import { CubeQueryConfig } from '../cube-query-config.types';
+export interface ActionDefinitionInterface {
+    tenant: TenantInterface;
+    eventType: string;
+    eventTopic: string;
+    destinationTopic: string;
+    actionIdentifier: string;
+    template: string;
+    fileNameTemplate: string | undefined;
+    queryConfig: CubeQueryConfig | undefined;
+}

package/libs/types/src/lib/management-api/chart.interface.d.ts

@@ -0,0 +1,9 @@
+export interface ChartInterface {
+    dashboards: Array<number>;
+    owners: Array<number>;
+    params: string;
+    slice_name: string;
+    viz_type: string;
+    id: string;
+    supersetId: number;
+}