@owox/backend 0.35.0 → 0.36.0

package/dist/src/data-marts/services/connector/connector-process-spawner.service.js

@@ -10,19 +10,83 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 var ConnectorProcessSpawnerService_1;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ConnectorProcessSpawnerService = void 0;
+exports.ConnectorProcessSpawnerService = exports.INHERITED_CONNECTOR_ENV_VARS = exports.TRUNCATED_OUTPUT_LINE = exports.MAX_CAPTURED_LINE_LENGTH = void 0;
+exports.inheritConnectorEnv = inheritConnectorEnv;
+exports.createCapturedLineBuffer = createCapturedLineBuffer;
 const common_1 = require("@nestjs/common");
 const cross_spawn_1 = require("cross-spawn");
 const graceful_shutdown_service_1 = require("../../../common/scheduler/services/graceful-shutdown.service");
-const MAX_CAPTURED_LINE_LENGTH = 1024 * 1024;
-const TRUNCATED_OUTPUT_LINE = '[TRUNCATED connector output line: exceeded 1048576 bytes]';
+exports.MAX_CAPTURED_LINE_LENGTH = 1024 * 1024;
+exports.TRUNCATED_OUTPUT_LINE = '[TRUNCATED connector output line: exceeded 1048576 bytes]';
+exports.INHERITED_CONNECTOR_ENV_VARS = [
+    'PATH',
+    'NODE_OPTIONS',
+    'NODE_EXTRA_CA_CERTS',
+    'SSL_CERT_FILE',
+    'SSL_CERT_DIR',
+    'HOME',
+    'USERPROFILE',
+    'TMPDIR',
+    'TMP',
+    'TEMP',
+    'SystemRoot',
+];
+const MANIFEST_RUN_INHERITED_ENV_VARS = [...exports.INHERITED_CONNECTOR_ENV_VARS, 'NODE_ENV'];
+function inheritConnectorEnv(names = exports.INHERITED_CONNECTOR_ENV_VARS) {
+    return Object.fromEntries(names.map(name => [name, process.env[name]]));
+}
+function createCapturedLineBuffer(onLine) {
+    let buffer = '';
+    let discardingOversizedLine = false;
+    const emit = (line) => {
+        onLine(line.endsWith('\r') ? line.slice(0, -1) : line);
+    };
+    return {
+        push: (chunk) => {
+            const parts = chunk.split('\n');
+            parts.forEach((part, index) => {
+                const isLastPart = index === parts.length - 1;
+                if (discardingOversizedLine) {
+                    if (!isLastPart) {
+                        discardingOversizedLine = false;
+                        buffer = '';
+                    }
+                    return;
+                }
+                if (buffer.length + part.length > exports.MAX_CAPTURED_LINE_LENGTH) {
+                    emit(exports.TRUNCATED_OUTPUT_LINE);
+                    buffer = '';
+                    discardingOversizedLine = isLastPart;
+                    return;
+                }
+                buffer += part;
+                if (!isLastPart) {
+                    emit(buffer);
+                    buffer = '';
+                }
+            });
+        },
+        flush: () => {
+            if (discardingOversizedLine) {
+                discardingOversizedLine = false;
+                buffer = '';
+                return;
+            }
+            if (!buffer) {
+                return;
+            }
+            emit(buffer);
+            buffer = '';
+        },
+    };
+}
 let ConnectorProcessSpawnerService = ConnectorProcessSpawnerService_1 = class ConnectorProcessSpawnerService {
     gracefulShutdownService;
     logger = new common_1.Logger(ConnectorProcessSpawnerService_1.name);
     constructor(gracefulShutdownService) {
         this.gracefulShutdownService = gracefulShutdownService;
     }
-    spawnConnector(datamartId, runId, configuration, runConfig, stdio, signal) {
+    spawnConnector(datamartId, runId, configuration, runConfig, stdio, signal, manifest) {
         return new Promise((resolve, reject) => {
             const spawnStdio = 'pipe';
             let logCapture = null;
@@ -34,13 +98,7 @@ let ConnectorProcessSpawnerService = ConnectorProcessSpawnerService_1 = class Co
             if (stdio && typeof stdio === 'object' && typeof stdio.onSpawn === 'function') {
                 onSpawn = stdio.onSpawn;
             }
-            const env = {
-                ...process.env,
-                OW_DATAMART_ID: datamartId,
-                OW_RUN_ID: runId,
-                OW_CONFIG: JSON.stringify(configuration.toObject()),
-                OW_RUN_CONFIG: JSON.stringify(runConfig.toObject()),
-            };
+            const env = this.buildChildEnv(datamartId, runId, configuration, runConfig, manifest);
             this.logger.log(`Spawning new process for connector runner execution for datamart ${datamartId} and run ${runId}`, { datamartId, runId });
             const runnerPath = require.resolve('@owox/connectors/runner');
             const node = (0, cross_spawn_1.spawn)('node', ['--no-deprecation', runnerPath], {
@@ -57,13 +115,15 @@ let ConnectorProcessSpawnerService = ConnectorProcessSpawnerService_1 = class Co
                 }
             }
             if (node.stdout && node.stderr) {
-                const stdoutBuffer = this.createLineBuffer(message => logCapture?.onStdout?.(message));
-                const stderrBuffer = this.createLineBuffer(message => logCapture?.onStderr?.(message));
-                node.stdout.on('data', data => {
-                    stdoutBuffer.push(data.toString());
+                const stdoutBuffer = createCapturedLineBuffer(message => logCapture?.onStdout?.(message));
+                const stderrBuffer = createCapturedLineBuffer(message => logCapture?.onStderr?.(message));
+                node.stdout.setEncoding('utf8');
+                node.stderr.setEncoding('utf8');
+                node.stdout.on('data', (data) => {
+                    stdoutBuffer.push(data);
                 });
-                node.stderr.on('data', data => {
-                    stderrBuffer.push(data.toString());
+                node.stderr.on('data', (data) => {
+                    stderrBuffer.push(data);
                 });
                 node.stdout.on('end', () => {
                     stdoutBuffer.flush();
@@ -122,50 +182,18 @@ let ConnectorProcessSpawnerService = ConnectorProcessSpawnerService_1 = class Co
             });
         });
     }
-    createLineBuffer(onLine) {
-        let buffer = '';
-        let discardingOversizedLine = false;
-        const emit = (line) => {
-            onLine(line.endsWith('\r') ? line.slice(0, -1) : line);
-        };
-        return {
-            push: (chunk) => {
-                const parts = chunk.split('\n');
-                parts.forEach((part, index) => {
-                    const isLastPart = index === parts.length - 1;
-                    if (discardingOversizedLine) {
-                        if (!isLastPart) {
-                            discardingOversizedLine = false;
-                            buffer = '';
-                        }
-                        return;
-                    }
-                    if (buffer.length + part.length > MAX_CAPTURED_LINE_LENGTH) {
-                        emit(TRUNCATED_OUTPUT_LINE);
-                        buffer = '';
-                        discardingOversizedLine = isLastPart;
-                        return;
-                    }
-                    buffer += part;
-                    if (!isLastPart) {
-                        emit(buffer);
-                        buffer = '';
-                    }
-                });
-            },
-            flush: () => {
-                if (discardingOversizedLine) {
-                    discardingOversizedLine = false;
-                    buffer = '';
-                    return;
-                }
-                if (!buffer) {
-                    return;
-                }
-                emit(buffer);
-                buffer = '';
-            },
+    buildChildEnv(datamartId, runId, configuration, runConfig, manifest) {
+        const env = {
+            ...(manifest ? inheritConnectorEnv(MANIFEST_RUN_INHERITED_ENV_VARS) : { ...process.env }),
+            OW_DATAMART_ID: datamartId,
+            OW_RUN_ID: runId,
+            OW_CONFIG: JSON.stringify(configuration.toObject()),
+            OW_RUN_CONFIG: JSON.stringify(runConfig.toObject()),
         };
+        if (manifest) {
+            env.OW_MANIFEST = JSON.stringify(manifest);
+        }
+        return env;
     }
 };
 exports.ConnectorProcessSpawnerService = ConnectorProcessSpawnerService;

package/dist/src/data-marts/services/connector/connector-secret.service.d.ts

@@ -2,6 +2,7 @@ import { ConnectorDefinition } from '../../dto/schemas/data-mart-table-definitio
 import { ConnectorService } from './connector.service';
 import { ConnectorSourceCredentialsService } from './connector-source-credentials.service';
 export declare const SECRET_MASK: "**********";
+export type SecretFieldsCache = Map<string, Promise<Set<string>>>;
 export declare class ConnectorSecretService {
     private readonly connectorService;
     private readonly connectorSourceCredentialsService;
@@ -20,12 +21,14 @@ export declare class ConnectorSecretService {
     extractAndSaveSecrets(dataMartId: string, projectId: string, connectorName: string, definition: ConnectorDefinition, userId?: string): Promise<ConnectorDefinition>;
     private maskRecursively;
     private mergeSecretsRecursively;
-    mask(definition: ConnectorDefinition | undefined): Promise<ConnectorDefinition | undefined>;
+    private maskWithoutSpecification;
+    private maskEveryValueRecursively;
+    mask(projectId: string | undefined, definition: ConnectorDefinition | undefined, secretFieldsCache?: SecretFieldsCache): Promise<ConnectorDefinition | undefined>;
     private injectMasksAtPaths;
-    mergeDefinitionSecrets(incoming: ConnectorDefinition, previous: ConnectorDefinition | undefined): Promise<ConnectorDefinition>;
+    mergeDefinitionSecrets(projectId: string, incoming: ConnectorDefinition, previous: ConnectorDefinition | undefined): Promise<ConnectorDefinition>;
     deleteOrphanedSecrets(dataMartId: string, currentDefinition: ConnectorDefinition, previousDefinition: ConnectorDefinition | undefined): Promise<void>;
     private collectSecretsIds;
-    mergeDefinitionSecretsFromSource(incoming: ConnectorDefinition, sourceDefinition: ConnectorDefinition): Promise<ConnectorDefinition>;
+    mergeDefinitionSecretsFromSource(projectId: string, incoming: ConnectorDefinition, sourceDefinition: ConnectorDefinition): Promise<ConnectorDefinition>;
     private hasSameRefreshToken;
     private getRefreshTokenValue;
 }

package/dist/src/data-marts/services/connector/connector-secret.service.js

@@ -16,6 +16,7 @@ const crypto_1 = require("crypto");
 const connector_service_1 = require("./connector.service");
 const connector_source_credentials_service_1 = require("./connector-source-credentials.service");
 const connectors_1 = require("@owox/connectors");
+const internal_helpers_1 = require("@owox/internal-helpers");
 exports.SECRET_MASK = '**********';
 const { GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD } = connectors_1.Core;
 let ConnectorSecretService = ConnectorSecretService_1 = class ConnectorSecretService {
@@ -29,8 +30,10 @@ let ConnectorSecretService = ConnectorSecretService_1 = class ConnectorSecretSer
     isSecretMask(value) {
         return typeof value === 'string' && value === exports.SECRET_MASK;
     }
-    async getAllSecretFieldNames(connectorName) {
-        const specification = await this.connectorService.getConnectorSpecification(connectorName);
+    async getAllSecretFieldNames(projectId, connectorName, version) {
+        const specification = projectId
+            ? await this.connectorService.resolveConnectorSpecification(projectId, connectorName, version)
+            : await this.connectorService.getConnectorSpecification(connectorName);
         const secretFields = new Set();
         const collectSecretFields = (fields) => {
             for (const field of fields) {
@@ -184,10 +187,22 @@ let ConnectorSecretService = ConnectorSecretService_1 = class ConnectorSecretSer
         }
     }
     async extractAndSaveSecrets(dataMartId, projectId, connectorName, definition, userId) {
-        const secretFieldNames = await this.getAllSecretFieldNames(connectorName);
+        const secretFieldNames = await this.getAllSecretFieldNames(projectId, connectorName, definition.connector.source.version);
         const hasGeneratedRefreshToken = this.hasGeneratedRefreshTokenRecursively(definition.connector.source.configuration);
         if (secretFieldNames.size === 0 && !hasGeneratedRefreshToken) {
-            return definition;
+            return {
+                ...definition,
+                connector: {
+                    ...definition.connector,
+                    source: {
+                        ...definition.connector.source,
+                        configuration: definition.connector.source.configuration.map(item => {
+                            const { _secrets_id: _dropped, ...rest } = item;
+                            return rest;
+                        }),
+                    },
+                },
+            };
         }
         const processedConfiguration = await Promise.all(definition.connector.source.configuration.map(async (item) => {
             const configItem = JSON.parse(JSON.stringify(item));
@@ -312,10 +327,55 @@ let ConnectorSecretService = ConnectorSecretService_1 = class ConnectorSecretSer
         }
         return incomingItem;
     }
-    async mask(definition) {
+    maskWithoutSpecification(definition) {
+        return {
+            ...definition,
+            connector: {
+                ...definition.connector,
+                source: {
+                    ...definition.connector.source,
+                    configuration: this.maskEveryValueRecursively(definition.connector.source.configuration),
+                },
+            },
+        };
+    }
+    maskEveryValueRecursively(value) {
+        if (Array.isArray(value)) {
+            return value.map(element => this.maskEveryValueRecursively(element));
+        }
+        if (value && typeof value === 'object') {
+            const masked = {};
+            for (const [key, nested] of Object.entries(value)) {
+                if (key === GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD) {
+                    continue;
+                }
+                masked[key] = key.startsWith('_') ? nested : this.maskEveryValueRecursively(nested);
+            }
+            return masked;
+        }
+        if (value === undefined || value === null || value === '') {
+            return value;
+        }
+        return exports.SECRET_MASK;
+    }
+    async mask(projectId, definition, secretFieldsCache) {
         if (!definition)
             return definition;
-        const secretFieldNames = await this.getAllSecretFieldNames(definition.connector.source.name);
+        let secretFieldNames;
+        try {
+            const { name, version } = definition.connector.source;
+            const key = `${name}@${version ?? 'active'}`;
+            let secretFields = secretFieldsCache?.get(key);
+            if (!secretFields) {
+                secretFields = this.getAllSecretFieldNames(projectId, name, version);
+                secretFieldsCache?.set(key, secretFields);
+            }
+            secretFieldNames = await secretFields;
+        }
+        catch (error) {
+            this.logger.warn(`Failed to resolve specification for connector "${definition.connector.source.name}" while masking; masking every configuration value`, (0, internal_helpers_1.castError)(error).stack);
+            return this.maskWithoutSpecification(definition);
+        }
         const hasGeneratedRefreshToken = this.hasGeneratedRefreshTokenRecursively(definition.connector.source.configuration);
         if (secretFieldNames.size === 0 && !hasGeneratedRefreshToken) {
             return definition;
@@ -366,13 +426,18 @@ let ConnectorSecretService = ConnectorSecretService_1 = class ConnectorSecretSer
             current[lastPart] = exports.SECRET_MASK;
         }
     }
-    async mergeDefinitionSecrets(incoming, previous) {
-        const secretFieldNames = await this.getAllSecretFieldNames(incoming.connector.source.name);
-        const previousConfiguration = previous?.connector?.source?.configuration || [];
+    async mergeDefinitionSecrets(projectId, incoming, previous) {
+        const secretFieldNames = await this.getAllSecretFieldNames(projectId, incoming.connector.source.name, incoming.connector.source.version);
+        const previousName = previous?.connector?.source?.name;
+        const switchedConnector = typeof previousName === 'string' &&
+            previousName.toLowerCase() !== incoming.connector.source.name.toLowerCase();
+        const previousConfiguration = switchedConnector
+            ? []
+            : previous?.connector?.source?.configuration || [];
         const mergedConfiguration = await Promise.all(incoming.connector.source.configuration.map(async (item) => {
             const incomingItem = (item || {});
             const usesSourceCredentials = this.hasSourceCredentialIdRecursively(incomingItem);
-            if (usesSourceCredentials) {
+            if (usesSourceCredentials || switchedConnector) {
                 delete incomingItem._secrets_id;
             }
             const itemId = incomingItem._id;
@@ -446,13 +511,13 @@ let ConnectorSecretService = ConnectorSecretService_1 = class ConnectorSecretSer
         }
         return secretsIds;
     }
-    async mergeDefinitionSecretsFromSource(incoming, sourceDefinition) {
+    async mergeDefinitionSecretsFromSource(projectId, incoming, sourceDefinition) {
         if (incoming.connector.source.name !== sourceDefinition.connector.source.name) {
             throw new Error(`Cannot copy secrets from different connector type. ` +
                 `Source: ${sourceDefinition.connector.source.name}, ` +
                 `Target: ${incoming.connector.source.name}`);
         }
-        const secretFieldNames = await this.getAllSecretFieldNames(incoming.connector.source.name);
+        const secretFieldNames = await this.getAllSecretFieldNames(projectId, incoming.connector.source.name, incoming.connector.source.version);
         const sourceSecretsIds = sourceDefinition.connector.source.configuration
             .map(item => item._secrets_id)
             .filter((id) => !!id);

package/dist/src/data-marts/services/connector/connector-source-config.service.js

@@ -26,7 +26,7 @@ let ConnectorSourceConfigService = ConnectorSourceConfigService_1 = class Connec
         const fieldsConfig = connector.source.fields
             .map(field => `${connector.source.node} ${field}`)
             .join(', ');
-        const configWithSecrets = await this.credentialInjector.injectSecrets(config, projectId);
+        const configWithSecrets = await this.credentialInjector.injectSecrets(config, projectId, connector.source.name);
         const configWithCredentials = await this.credentialInjector.injectOAuthCredentials(configWithSecrets, connector.source.name, projectId);
         return new SourceConfigDto({
             name: connector.source.name,

package/dist/src/data-marts/services/connector/connector-source-credentials.service.d.ts

@@ -1,7 +1,12 @@
 import { Repository } from 'typeorm';
 import { ConnectorSourceCredentials } from '../../entities/connector-source-credentials.entity';
+export interface CredentialFieldUpdateResult {
+    credentials: ConnectorSourceCredentials;
+    updated: boolean;
+}
 export declare class ConnectorSourceCredentialsService {
     private readonly connectorSourceCredentialsRepository;
+    private readonly logger;
     constructor(connectorSourceCredentialsRepository: Repository<ConnectorSourceCredentials>);
     createCredentials(projectId: string, userId: string, connectorName: string, credentials: Record<string, unknown>, expiresAt: Date | null, user?: {
         id?: string;
@@ -10,7 +15,6 @@ export declare class ConnectorSourceCredentialsService {
         picture?: string;
     }): Promise<ConnectorSourceCredentials>;
     getCredentialsById(id: string): Promise<ConnectorSourceCredentials | null>;
-    getCredentialsByConnectorName(connectorName: string): Promise<ConnectorSourceCredentials[]>;
     updateCredentials(id: string, credentials: Record<string, unknown>, expiresAt?: Date | null): Promise<ConnectorSourceCredentials>;
     deleteCredentialsByIdsAndDataMart(ids: string[], dataMartId: string): Promise<number>;
     deleteCredentialsByConnectorName(connectorName: string): Promise<void>;
@@ -19,7 +23,7 @@ export declare class ConnectorSourceCredentialsService {
     createSecretsForConfig(projectId: string, connectorName: string, dataMartId: string, configId: string, secrets: Record<string, unknown>, userId?: string): Promise<ConnectorSourceCredentials>;
     getSecretsByDataMartAndConfig(dataMartId: string, configId: string): Promise<ConnectorSourceCredentials | null>;
     updateSecretsForConfig(id: string, projectId: string, secrets: Record<string, unknown>): Promise<ConnectorSourceCredentials>;
-    updateCredentialFields(id: string, projectId: string, updates: Record<string, unknown>, expectedCurrentValues?: Record<string, unknown>): Promise<ConnectorSourceCredentials>;
+    updateCredentialFields(id: string, projectId: string, updates: Record<string, unknown>, expectedCurrentValues?: Record<string, unknown>): Promise<CredentialFieldUpdateResult>;
     private getJsonExtractTextExpression;
     private getDatabaseType;
     private updateCredentialsJson;

package/dist/src/data-marts/services/connector/connector-source-credentials.service.js

@@ -11,6 +11,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 var __param = (this && this.__param) || function (paramIndex, decorator) {
     return function (target, key) { decorator(target, key, paramIndex); }
 };
+var ConnectorSourceCredentialsService_1;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ConnectorSourceCredentialsService = void 0;
 const common_1 = require("@nestjs/common");
@@ -20,8 +21,9 @@ const connector_source_credentials_entity_1 = require("../../entities/connector-
 const connectors_1 = require("@owox/connectors");
 const { GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD } = connectors_1.Core;
 const SECRET_MASK = '**********';
-let ConnectorSourceCredentialsService = class ConnectorSourceCredentialsService {
+let ConnectorSourceCredentialsService = ConnectorSourceCredentialsService_1 = class ConnectorSourceCredentialsService {
     connectorSourceCredentialsRepository;
+    logger = new common_1.Logger(ConnectorSourceCredentialsService_1.name);
     constructor(connectorSourceCredentialsRepository) {
         this.connectorSourceCredentialsRepository = connectorSourceCredentialsRepository;
     }
@@ -41,12 +43,6 @@ let ConnectorSourceCredentialsService = class ConnectorSourceCredentialsService
             where: { id },
         });
     }
-    async getCredentialsByConnectorName(connectorName) {
-        return await this.connectorSourceCredentialsRepository.find({
-            where: { connectorName },
-            order: { createdAt: 'DESC' },
-        });
-    }
     async updateCredentials(id, credentials, expiresAt) {
         const existingCredentials = await this.getCredentialsById(id);
         if (!existingCredentials) {
@@ -160,16 +156,22 @@ let ConnectorSourceCredentialsService = class ConnectorSourceCredentialsService
         }
         const updateResult = await queryBuilder.execute();
         if (updateResult.affected === 0) {
-            return existing;
+            this.logger.warn(`Rotated credential was not persisted: the stored value changed while this run was ` +
+                `executing, so the newly issued token was discarded. Authentication may fail on the ` +
+                `next run until the source is reconnected.`, { credentialId: id, projectId });
+            return { credentials: existing, updated: false };
         }
-        const updated = await this.getCredentialsById(id);
-        return (updated ?? {
-            ...existing,
-            credentials: {
-                ...existing.credentials,
-                [GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD]: generatedRefreshToken,
+        const stored = await this.getCredentialsById(id);
+        return {
+            credentials: stored ?? {
+                ...existing,
+                credentials: {
+                    ...existing.credentials,
+                    [GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD]: generatedRefreshToken,
+                },
             },
-        });
+            updated: true,
+        };
     }
     getJsonExtractTextExpression(column, path) {
         const databaseType = this.getDatabaseType();
@@ -261,7 +263,7 @@ let ConnectorSourceCredentialsService = class ConnectorSourceCredentialsService
     }
 };
 exports.ConnectorSourceCredentialsService = ConnectorSourceCredentialsService;
-exports.ConnectorSourceCredentialsService = ConnectorSourceCredentialsService = __decorate([
+exports.ConnectorSourceCredentialsService = ConnectorSourceCredentialsService = ConnectorSourceCredentialsService_1 = __decorate([
     (0, common_1.Injectable)(),
     __param(0, (0, typeorm_1.InjectRepository)(connector_source_credentials_entity_1.ConnectorSourceCredentials)),
     __metadata("design:paramtypes", [typeorm_2.Repository])

package/dist/src/data-marts/services/connector/connector-specification.mapper.d.ts

@@ -0,0 +1,47 @@
+import { Core } from '@owox/connectors';
+interface ConnectorSpecificationOneOf {
+    label: string;
+    value: string;
+    requiredType: string;
+    attributes?: Core.CONFIG_ATTRIBUTES[];
+    oauthParams?: Record<string, unknown>;
+    items: Record<string, ConnectorConfigField>;
+}
+interface ConnectorConfigField {
+    description: string;
+    label: string;
+    default: unknown;
+    requiredType: string;
+    isRequired: boolean;
+    options?: unknown[];
+    placeholder?: string;
+    minimum?: number;
+    attributes?: Core.CONFIG_ATTRIBUTES[];
+    optionsDependsOn?: string[];
+    oneOf?: ConnectorSpecificationOneOf[];
+}
+export interface ConnectorConfig {
+    [key: string]: ConnectorConfigField;
+}
+export declare function mapConnectorSpecification(config: ConnectorConfig): {
+    oneOf: {
+        label: string;
+        value: string;
+        requiredType: string;
+        attributes: any[] | undefined;
+        oauthParams: Record<string, unknown> | undefined;
+        items: Record<string, unknown>;
+    }[] | undefined;
+    name: string;
+    title: string;
+    description: string;
+    default: unknown;
+    requiredType: string;
+    required: boolean;
+    options: unknown[] | undefined;
+    placeholder: string | undefined;
+    minimum: number | undefined;
+    attributes: any[] | undefined;
+    optionsDependsOn: string[] | undefined;
+}[];
+export {};

package/dist/src/data-marts/services/connector/connector-specification.mapper.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapConnectorSpecification = mapConnectorSpecification;
+const connectors_1 = require("@owox/connectors");
+function mapConnectorSpecification(config) {
+    const result = Object.keys(config).map(key => {
+        const item = {
+            ...mapConfigFieldToSchema(key, config[key]),
+            oneOf: config[key].oneOf?.map(oneOf => {
+                return {
+                    label: oneOf.label,
+                    value: oneOf.value,
+                    requiredType: oneOf.requiredType,
+                    attributes: oneOf.attributes,
+                    oauthParams: oneOf.oauthParams,
+                    items: Object.entries(oneOf.items).reduce((acc, [itemKey, itemValue]) => {
+                        acc[itemKey] = mapConfigFieldToSchema(itemKey, itemValue);
+                        return acc;
+                    }, {}),
+                };
+            }),
+        };
+        return item;
+    });
+    return result;
+}
+function mapConfigFieldToSchema(name, field) {
+    const item = {
+        name,
+        title: field.label,
+        description: field.description,
+        default: field.default,
+        requiredType: field.requiredType,
+        required: field.isRequired,
+        options: field.options,
+        placeholder: field.placeholder,
+        minimum: field.minimum,
+        attributes: field.attributes,
+        optionsDependsOn: field.optionsDependsOn,
+    };
+    if (!(field.attributes ?? []).includes(connectors_1.Core.CONFIG_ATTRIBUTES.SECRET)) {
+        return item;
+    }
+    return { ...item, default: undefined, options: undefined, placeholder: undefined };
+}
+//# sourceMappingURL=connector-specification.mapper.js.map

package/dist/src/data-marts/services/connector/connector-test.service.d.ts

@@ -0,0 +1,43 @@
+import { ConfigService } from '@nestjs/config';
+export declare const TEST_DATE_WINDOW_DAYS = 2;
+export declare const MAX_TEST_OUTPUT_LENGTH: number;
+export declare const CONNECTOR_TEST_CONCURRENCY_LIMIT_CODE = "CONNECTOR_TEST_CONCURRENCY_LIMIT";
+export interface ConnectorTestResult {
+    rows: Record<string, unknown>[];
+    logs: string[];
+    error: string | null;
+    sample: Record<string, unknown>[];
+}
+export interface ConnectorTestRequest {
+    projectId: string;
+    manifest: Record<string, unknown>;
+    node: string;
+    configuration: Record<string, unknown>;
+    maxRows?: number;
+    maxPages?: number;
+    timeoutMs?: number;
+}
+export interface ConnectorTestHooks {
+    hang?: boolean;
+    env?: Record<string, string>;
+}
+export declare class ConnectorTestService {
+    private readonly configService?;
+    private readonly activeTestsByProject;
+    private activeTestsTotal;
+    constructor(configService?: ConfigService | undefined);
+    wrapConfig(configuration: Record<string, unknown>): Record<string, {
+        value: unknown;
+    }>;
+    pruneToNode(manifest: Record<string, unknown>, node: string): Record<string, unknown>;
+    selectNodeFields(manifest: Record<string, unknown>, node: string): string;
+    private manifestParseError;
+    private assertFitsChildEnv;
+    protected runnerPath(): string;
+    private stopChild;
+    private testDateWindow;
+    private limit;
+    private acquireTestSlot;
+    runTest(args: ConnectorTestRequest, hooks?: ConnectorTestHooks): Promise<ConnectorTestResult>;
+    private execute;
+}