@owox/backend 0.35.0 → 0.36.0

package/dist/src/data-marts/services/connector/connector-test.service.js

@@ -0,0 +1,343 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConnectorTestService = exports.CONNECTOR_TEST_CONCURRENCY_LIMIT_CODE = exports.MAX_TEST_OUTPUT_LENGTH = exports.TEST_DATE_WINDOW_DAYS = void 0;
+const common_1 = require("@nestjs/common");
+const config_1 = require("@nestjs/config");
+const cross_spawn_1 = require("cross-spawn");
+const connectors_1 = require("@owox/connectors");
+const internal_helpers_1 = require("@owox/internal-helpers");
+const concurrency_limit_exceeded_exception_1 = require("../../../common/exceptions/concurrency-limit-exceeded.exception");
+const custom_connector_dto_1 = require("../../dto/presentation/custom-connector.dto");
+const connector_process_spawner_service_1 = require("./connector-process-spawner.service");
+const SAMPLE_ONLY_FIELD = '__owox_sample__';
+const DEFAULT_TIMEOUT_MS = 20000;
+exports.TEST_DATE_WINDOW_DAYS = 2;
+exports.MAX_TEST_OUTPUT_LENGTH = 16 * 1024 * 1024;
+const DEFAULT_MAX_TESTS_PER_PROJECT = 3;
+const DEFAULT_MAX_TESTS_TOTAL = 10;
+exports.CONNECTOR_TEST_CONCURRENCY_LIMIT_CODE = 'CONNECTOR_TEST_CONCURRENCY_LIMIT';
+const CHILD_EXIT_GRACE_MS = 1000;
+function readRunFailureEnvelope(line) {
+    let envelope;
+    try {
+        envelope = JSON.parse(line);
+    }
+    catch {
+        return null;
+    }
+    if (envelope?.type === 'addWarningToCurrentStatus' && typeof envelope.warning === 'string') {
+        return envelope.warning;
+    }
+    if (envelope?.type === 'error' && typeof envelope.error === 'string') {
+        return envelope.error.split('\n')[0].replace(/^\w*Error: /, '');
+    }
+    return null;
+}
+let ConnectorTestService = class ConnectorTestService {
+    configService;
+    activeTestsByProject = new Map();
+    activeTestsTotal = 0;
+    constructor(configService) {
+        this.configService = configService;
+    }
+    wrapConfig(configuration) {
+        return Object.fromEntries(Object.entries(configuration).map(([k, v]) => [k, { value: v }]));
+    }
+    pruneToNode(manifest, node) {
+        const nodes = (manifest.nodes ?? {});
+        if (!nodes[node]) {
+            throw new common_1.BadRequestException(`Unknown node "${node}"`);
+        }
+        return { ...manifest, nodes: { [node]: nodes[node] } };
+    }
+    selectNodeFields(manifest, node) {
+        const nodes = (manifest.nodes ?? {});
+        const fields = nodes[node]?.fields ?? {};
+        return Object.keys(fields)
+            .map(f => `${node} ${f}`)
+            .join(', ');
+    }
+    manifestParseError(manifest) {
+        try {
+            new connectors_1.Core.ManifestParser().parse(JSON.stringify(manifest));
+            return null;
+        }
+        catch (e) {
+            return (0, internal_helpers_1.castError)(e).message;
+        }
+    }
+    assertFitsChildEnv(label, serialized) {
+        const sizeBytes = Buffer.byteLength(serialized, 'utf8');
+        if (sizeBytes > custom_connector_dto_1.MAX_MANIFEST_SIZE_BYTES) {
+            throw new common_1.BadRequestException(`Connector test ${label} is ${sizeBytes} bytes, over the ${custom_connector_dto_1.MAX_MANIFEST_SIZE_BYTES}-byte limit. ` +
+                `It is handed to the connector runner as an environment variable, which the operating ` +
+                `system refuses past that size.`);
+        }
+    }
+    runnerPath() {
+        return require.resolve('@owox/connectors/runner');
+    }
+    stopChild(child) {
+        if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) {
+            return Promise.resolve();
+        }
+        return new Promise(resolve => {
+            let done = false;
+            const settle = () => {
+                if (done)
+                    return;
+                done = true;
+                clearTimeout(killTimer);
+                clearTimeout(giveUpTimer);
+                resolve();
+            };
+            child.once('close', settle);
+            child.once('error', settle);
+            const killTimer = setTimeout(() => {
+                try {
+                    child.kill('SIGKILL');
+                }
+                catch {
+                }
+            }, CHILD_EXIT_GRACE_MS);
+            const giveUpTimer = setTimeout(settle, CHILD_EXIT_GRACE_MS * 2);
+            try {
+                child.kill('SIGTERM');
+            }
+            catch {
+                settle();
+            }
+        });
+    }
+    testDateWindow() {
+        const todayMs = Date.now();
+        return {
+            start: new Date(todayMs - (exports.TEST_DATE_WINDOW_DAYS - 1) * 86400000).toISOString().split('T')[0],
+            end: new Date(todayMs).toISOString().split('T')[0],
+        };
+    }
+    limit(key, fallback) {
+        const configured = Number(this.configService?.get(key, fallback) ?? fallback);
+        return Number.isFinite(configured) && configured >= 1 ? Math.floor(configured) : fallback;
+    }
+    acquireTestSlot(projectId) {
+        const perProject = this.limit('MAX_CONNECTOR_TESTS_PER_PROJECT', DEFAULT_MAX_TESTS_PER_PROJECT);
+        const total = this.limit('MAX_CONNECTOR_TESTS_TOTAL', DEFAULT_MAX_TESTS_TOTAL);
+        const retryAfterSeconds = Math.ceil(DEFAULT_TIMEOUT_MS / 1000);
+        const active = this.activeTestsByProject.get(projectId) ?? 0;
+        if (active >= perProject) {
+            throw new concurrency_limit_exceeded_exception_1.ConcurrencyLimitExceededException(`This project already has ${perProject} connector test${perProject === 1 ? '' : 's'} running. ` +
+                `Wait for one to finish, then try again.`, { scope: 'project', limit: perProject, retryAfterSeconds }, exports.CONNECTOR_TEST_CONCURRENCY_LIMIT_CODE);
+        }
+        if (this.activeTestsTotal >= total) {
+            throw new concurrency_limit_exceeded_exception_1.ConcurrencyLimitExceededException(`This deployment already has ${total} connector tests running. Try again in a moment.`, { scope: 'deployment', limit: total, retryAfterSeconds }, exports.CONNECTOR_TEST_CONCURRENCY_LIMIT_CODE);
+        }
+        this.activeTestsByProject.set(projectId, active + 1);
+        this.activeTestsTotal += 1;
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            this.activeTestsTotal -= 1;
+            const remaining = (this.activeTestsByProject.get(projectId) ?? 1) - 1;
+            if (remaining > 0)
+                this.activeTestsByProject.set(projectId, remaining);
+            else
+                this.activeTestsByProject.delete(projectId);
+        };
+    }
+    async runTest(args, hooks = {}) {
+        this.assertFitsChildEnv('manifest', JSON.stringify(args.manifest));
+        this.assertFitsChildEnv('configuration', JSON.stringify(this.wrapConfig(args.configuration ?? {})));
+        const release = this.acquireTestSlot(args.projectId);
+        try {
+            const result = await this.execute(args, hooks);
+            return result;
+        }
+        finally {
+            release();
+        }
+    }
+    async execute(args, hooks) {
+        const maxRows = args.maxRows ?? 25;
+        const maxPages = args.maxPages ?? 1;
+        const timeoutMs = args.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        const pruned = this.pruneToNode(args.manifest, args.node);
+        const prunedError = this.manifestParseError(pruned);
+        if (prunedError) {
+            throw new common_1.BadRequestException(`Invalid manifest: ${prunedError}`);
+        }
+        const untestedNodeError = this.manifestParseError(args.manifest);
+        const sourceConfig = this.wrapConfig(args.configuration);
+        let sampleOnly = false;
+        if (sourceConfig.Fields === undefined) {
+            const fieldsSelection = this.selectNodeFields(pruned, args.node);
+            if (fieldsSelection) {
+                sourceConfig.Fields = { value: fieldsSelection };
+            }
+            else {
+                sampleOnly = true;
+                sourceConfig.Fields = { value: `${args.node} ${SAMPLE_ONLY_FIELD}` };
+            }
+        }
+        const dateWindow = this.testDateWindow();
+        sourceConfig.LastRequestedDate = { value: dateWindow.start };
+        sourceConfig.ReimportLookbackWindow = { value: 0 };
+        const env = {
+            ...(0, connector_process_spawner_service_1.inheritConnectorEnv)(),
+            OW_DATAMART_ID: 'test',
+            OW_RUN_ID: 'test',
+            OW_TEST: '1',
+            OW_TEST_MAX_ROWS: String(maxRows),
+            OW_TEST_MAX_PAGES: String(maxPages),
+            OW_MANIFEST: JSON.stringify(pruned),
+            OW_CONFIG: JSON.stringify({
+                source: { name: String(args.manifest.name ?? 'Custom'), config: sourceConfig },
+                storage: { name: 'LiveTest', config: {} },
+            }),
+            OW_RUN_CONFIG: JSON.stringify({ type: 'INCREMENTAL', data: [], state: {} }),
+        };
+        if (hooks.hang)
+            env.FAKE_HANG = '1';
+        if (hooks.env)
+            Object.assign(env, hooks.env);
+        const marker = connectors_1.Core.TEST_ROW_MARKER;
+        const rows = [];
+        const logs = [];
+        const errorLogs = [];
+        let runFailure = null;
+        let sample = [];
+        return new Promise(resolve => {
+            const child = (0, cross_spawn_1.spawn)('node', ['--no-deprecation', this.runnerPath()], {
+                env,
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+            let settled = false;
+            const finish = (error) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                stderrBuffer.flush();
+                const captured = rows.slice(0, maxRows);
+                if (error === null)
+                    error = runFailure;
+                if (error === null && captured.length === 0)
+                    error = errorLogs[0] ?? null;
+                if (error === null && captured.length === 0) {
+                    logs.push(`Test produced 0 records: the request completed without error but recordSelector.recordPath matched no rows.` +
+                        (sample.length > 0
+                            ? ` A raw response sample WAS received (${sample.length} record(s)), so recordPath is the likely culprit — compare it against the sample shape.`
+                            : ` Check recordPath, the request path/queryParameters, and any filters.`) +
+                        ` If this node is fetched by date, note that a live test samples only ` +
+                        `${dateWindow.start} to ${dateWindow.end}; a window with no data upstream ` +
+                        `returns 0 records too.`);
+                }
+                if (untestedNodeError) {
+                    logs.push(`This test ran only the node "${args.node}". Publishing validates every node, and ` +
+                        `another node in this manifest cannot be published: ${untestedNodeError}`);
+                }
+                const result = {
+                    rows: sampleOnly ? [] : captured,
+                    logs,
+                    error,
+                    sample,
+                };
+                void this.stopChild(child).then(() => resolve(result));
+            };
+            const timer = setTimeout(() => finish(`Test run timed out after ${timeoutMs}ms`), timeoutMs);
+            let held = 0;
+            let overBudget = false;
+            const admit = (line) => {
+                if (overBudget)
+                    return false;
+                if (held + line.length <= exports.MAX_TEST_OUTPUT_LENGTH) {
+                    held += line.length;
+                    return true;
+                }
+                overBudget = true;
+                logs.push(`Stopped reading the test output at ${exports.MAX_TEST_OUTPUT_LENGTH / 1024 / 1024} MiB: ` +
+                    `the rows shown are the ones that arrived by then.`);
+                finish(null);
+                return false;
+            };
+            const onLine = (line) => {
+                if (!line || !admit(line))
+                    return;
+                if (line.startsWith(marker)) {
+                    try {
+                        rows.push(JSON.parse(line.slice(marker.length)));
+                    }
+                    catch {
+                    }
+                    if (rows.length >= maxRows)
+                        finish(null);
+                    return;
+                }
+                try {
+                    const evt = JSON.parse(line);
+                    if (evt && evt.type === 'SAMPLE') {
+                        sample = Array.isArray(evt.records) ? evt.records : [];
+                        return;
+                    }
+                    if (evt &&
+                        evt.type === 'LOG' &&
+                        evt.level === 'error' &&
+                        typeof evt.message === 'string') {
+                        errorLogs.push(evt.message);
+                    }
+                    if (evt &&
+                        evt.type === 'CONTROL' &&
+                        evt.action === 'failed' &&
+                        typeof evt.error === 'string') {
+                        runFailure ??= evt.error;
+                    }
+                }
+                catch {
+                }
+                logs.push(line);
+            };
+            const stdoutBuffer = (0, connector_process_spawner_service_1.createCapturedLineBuffer)(onLine);
+            const stderrBuffer = (0, connector_process_spawner_service_1.createCapturedLineBuffer)(line => {
+                if (!line || !admit(line))
+                    return;
+                logs.push(line);
+                const reported = readRunFailureEnvelope(line);
+                if (reported !== null)
+                    runFailure ??= reported;
+            });
+            child.stdout?.setEncoding('utf8');
+            child.stderr?.setEncoding('utf8');
+            child.stdout?.on('data', (chunk) => stdoutBuffer.push(chunk));
+            child.stderr?.on('data', (chunk) => stderrBuffer.push(chunk));
+            child.on('close', code => {
+                if (settled)
+                    return;
+                stdoutBuffer.flush();
+                stderrBuffer.flush();
+                finish(code === 0 ? null : `Test process exited with code ${String(code)}`);
+            });
+            child.on('error', err => finish(err.message));
+        });
+    }
+};
+exports.ConnectorTestService = ConnectorTestService;
+exports.ConnectorTestService = ConnectorTestService = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Optional)()),
+    __metadata("design:paramtypes", [config_1.ConfigService])
+], ConnectorTestService);
+//# sourceMappingURL=connector-test.service.js.map

package/dist/src/data-marts/services/connector/connector.service.d.ts

@@ -2,15 +2,22 @@ import { ConnectorDefinition } from '../../connector-types/connector-definition'
 import { ConnectorSpecification, ConnectorSpecificationItem } from '../../connector-types/connector-specification';
 import { ConnectorFieldsSchema } from '../../connector-types/connector-fields-schema';
 import { ConnectorSourceCredentialsService } from './connector-source-credentials.service';
+import { ConnectorDefinitionService } from './connector-definition.service';
 import type { ConnectorCapabilities } from '../../connector-types/connector-capabilities';
 export declare class ConnectorService {
     private readonly connectorSourceCredentialsService;
+    private readonly connectorDefinitionService;
     private readonly logger;
-    constructor(connectorSourceCredentialsService: ConnectorSourceCredentialsService);
+    constructor(connectorSourceCredentialsService: ConnectorSourceCredentialsService, connectorDefinitionService: ConnectorDefinitionService);
     getAvailableConnectors(): Promise<ConnectorDefinition[]>;
     getConnectorCapabilities(connectorName: string): ConnectorCapabilities;
+    resolveConnectorCapabilities(projectId: string, connectorName: string, version?: number): Promise<ConnectorCapabilities>;
+    private resolveBundledOrCustom;
     getConnectorSpecification(connectorName: string): Promise<ConnectorSpecification>;
+    resolveConnectorSpecification(projectId: string, connectorName: string, version?: number): Promise<ConnectorSpecification>;
     getConnectorFieldsSchema(connectorName: string): Promise<ConnectorFieldsSchema>;
+    getSpecificationFromManifest(manifest: Record<string, unknown>): ConnectorSpecification;
+    getFieldsSchemaFromManifest(manifest: Record<string, unknown>): ConnectorFieldsSchema;
     getOAuthUiVariables(connectorName: string, fieldPath: string): Promise<Record<string, unknown>>;
     getOAuthUiVariablesExpanded(connectorName: string, fieldPath: string): Promise<Record<string, unknown>>;
     isOAuthEnabled(connectorName: string, fieldPath: string): Promise<boolean>;
@@ -34,5 +41,4 @@ export declare class ConnectorService {
     private validateConnectorExists;
     private createConnectorSource;
     private getConnectorManifest;
-    private mapConfigToSchema;
 }

package/dist/src/data-marts/services/connector/connector.service.js

@@ -16,12 +16,22 @@ const connectors_1 = require("@owox/connectors");
 const connector_specification_1 = require("../../connector-types/connector-specification");
 const connector_fields_schema_1 = require("../../connector-types/connector-fields-schema");
 const connector_source_credentials_service_1 = require("./connector-source-credentials.service");
+const connector_definition_service_1 = require("./connector-definition.service");
 const connector_fields_schema_mapper_1 = require("./connector-fields-schema.mapper");
+const connector_specification_mapper_1 = require("./connector-specification.mapper");
+const declarative_manifest_schemas_1 = require("./declarative-manifest-schemas");
+const connector_credential_boundary_error_1 = require("../../errors/connector-credential-boundary.error");
+const NO_CAPABILITIES = Object.freeze({
+    singleConfiguration: false,
+    copySecretsByValue: false,
+});
 let ConnectorService = ConnectorService_1 = class ConnectorService {
     connectorSourceCredentialsService;
+    connectorDefinitionService;
     logger = new common_1.Logger(ConnectorService_1.name);
-    constructor(connectorSourceCredentialsService) {
+    constructor(connectorSourceCredentialsService, connectorDefinitionService) {
         this.connectorSourceCredentialsService = connectorSourceCredentialsService;
+        this.connectorDefinitionService = connectorDefinitionService;
     }
     async getAvailableConnectors() {
         return connectors_1.AvailableConnectors.map(connector => {
@@ -29,9 +39,9 @@ let ConnectorService = ConnectorService_1 = class ConnectorService {
             return {
                 name: connector,
                 title: manifest.title,
-                description: manifest.description,
-                logo: manifest.logo,
-                docUrl: manifest.docUrl,
+                description: manifest.description ?? null,
+                logo: manifest.logo ?? null,
+                docUrl: manifest.docUrl ?? null,
             };
         });
     }
@@ -43,12 +53,28 @@ let ConnectorService = ConnectorService_1 = class ConnectorService {
             copySecretsByValue: capabilities?.copySecretsByValue === true,
         };
     }
+    async resolveConnectorCapabilities(projectId, connectorName, version) {
+        return this.resolveBundledOrCustom(projectId, connectorName, version, name => this.getConnectorCapabilities(name), () => NO_CAPABILITIES);
+    }
+    async resolveBundledOrCustom(projectId, connectorName, version, fromBundled, fromManifest) {
+        if (Object.keys(connectors_1.Connectors).includes(connectorName)) {
+            return fromBundled(connectorName);
+        }
+        const manifest = await this.connectorDefinitionService.tryResolveManifest(projectId, connectorName, version);
+        if (manifest) {
+            return fromManifest(manifest);
+        }
+        return fromBundled(connectorName);
+    }
     async getConnectorSpecification(connectorName) {
         this.validateConnectorExists(connectorName);
         const source = this.createConnectorSource(connectorName);
-        const configSchema = this.mapConfigToSchema(source.config);
+        const configSchema = (0, connector_specification_mapper_1.mapConnectorSpecification)(source.parameters);
         return connector_specification_1.ConnectorSpecification.parse(configSchema);
     }
+    async resolveConnectorSpecification(projectId, connectorName, version) {
+        return this.resolveBundledOrCustom(projectId, connectorName, version, name => this.getConnectorSpecification(name), manifest => this.getSpecificationFromManifest(manifest));
+    }
     async getConnectorFieldsSchema(connectorName) {
         this.validateConnectorExists(connectorName);
         const sourceInstance = this.createConnectorSource(connectorName);
@@ -63,6 +89,12 @@ let ConnectorService = ConnectorService_1 = class ConnectorService {
         const fieldsSchema = (0, connector_fields_schema_mapper_1.mapConnectorFieldsSchema)(sourceFieldsSchema);
         return connector_fields_schema_1.ConnectorFieldsSchema.parse(fieldsSchema);
     }
+    getSpecificationFromManifest(manifest) {
+        return (0, declarative_manifest_schemas_1.specificationFromManifest)(manifest);
+    }
+    getFieldsSchemaFromManifest(manifest) {
+        return (0, declarative_manifest_schemas_1.fieldsSchemaFromManifest)(manifest);
+    }
     async getOAuthUiVariables(connectorName, fieldPath) {
         const specification = await this.getConnectorSpecification(connectorName);
         const paths = fieldPath.split('.');
@@ -130,10 +162,10 @@ let ConnectorService = ConnectorService_1 = class ConnectorService {
             throw new Error(`Credential with ID ${credentialId} not found`);
         }
         if (credential.projectId !== projectId) {
-            throw new Error(`Credential with ID ${credentialId} not found`);
+            throw new connector_credential_boundary_error_1.ConnectorCredentialBoundaryError(`Credential with ID ${credentialId} not found`);
         }
         if (credential.connectorName !== connectorName) {
-            throw new Error(`Credential belongs to connector ${credential.connectorName}, not ${connectorName}`);
+            throw new connector_credential_boundary_error_1.ConnectorCredentialBoundaryError(`Credential belongs to connector ${credential.connectorName}, not ${connectorName}`);
         }
         const connector = this.createConnectorSource(connectorName);
         const oauthVariables = await this.getOAuthVariablesForRefresh(connectorName, configuration);
@@ -262,61 +294,40 @@ let ConnectorService = ConnectorService_1 = class ConnectorService {
         }
     }
     createConnectorSource(connectorName) {
-        const source = connectors_1.Connectors[connectorName][`${connectorName}Source`];
-        return new source(new connectors_1.Core.AbstractConfig({}));
+        const context = new connectors_1.Core.AbstractContext({
+            source: { name: connectorName, config: {} },
+            storage: { name: 'unused', config: {} },
+            runConfig: {},
+            env: { datamartId: null, runId: null },
+        });
+        const SourceClass = connectors_1.Connectors[connectorName][`${connectorName}Source`];
+        if (SourceClass) {
+            return new SourceClass(context);
+        }
+        const manifest = connectors_1.Connectors[connectorName].manifest;
+        if (manifest && manifest.nodes) {
+            let model;
+            try {
+                model = new connectors_1.Core.ManifestParser().parse(JSON.stringify(manifest));
+            }
+            catch (e) {
+                this.logger.error(`Failed to parse declarative manifest for '${connectorName}': ${e.message}`);
+                throw new common_1.InternalServerErrorException(`Connector '${connectorName}' has an invalid declarative manifest`);
+            }
+            return new connectors_1.Core.DeclarativeSource(context, model);
+        }
+        this.logger.error(`Connector '${connectorName}' has neither a '${connectorName}Source' class nor a declarative manifest`);
+        throw new common_1.InternalServerErrorException(`Connector '${connectorName}' is misconfigured`);
     }
     getConnectorManifest(connectorName) {
         const manifest = connectors_1.Connectors[connectorName].manifest;
         return manifest;
     }
-    mapConfigToSchema(config) {
-        const result = Object.keys(config).map(key => {
-            const item = {
-                name: key,
-                title: config[key].label,
-                description: config[key].description,
-                default: config[key].default,
-                requiredType: config[key].requiredType,
-                required: config[key].isRequired,
-                options: config[key].options,
-                placeholder: config[key].placeholder,
-                minimum: config[key].minimum,
-                attributes: config[key].attributes,
-                optionsDependsOn: config[key].optionsDependsOn,
-                oneOf: config[key].oneOf?.map(oneOf => {
-                    return {
-                        label: oneOf.label,
-                        value: oneOf.value,
-                        requiredType: oneOf.requiredType,
-                        attributes: oneOf.attributes,
-                        oauthParams: oneOf.oauthParams,
-                        items: Object.entries(oneOf.items).reduce((acc, [itemKey, itemValue]) => {
-                            acc[itemKey] = {
-                                name: itemKey,
-                                title: itemValue.label,
-                                description: itemValue.description,
-                                default: itemValue.default,
-                                requiredType: itemValue.requiredType,
-                                required: itemValue.isRequired,
-                                options: itemValue.options,
-                                placeholder: itemValue.placeholder,
-                                minimum: itemValue.minimum,
-                                attributes: itemValue.attributes,
-                                optionsDependsOn: itemValue.optionsDependsOn,
-                            };
-                            return acc;
-                        }, {}),
-                    };
-                }),
-            };
-            return item;
-        });
-        return result;
-    }
 };
 exports.ConnectorService = ConnectorService;
 exports.ConnectorService = ConnectorService = ConnectorService_1 = __decorate([
     (0, common_1.Injectable)(),
-    __metadata("design:paramtypes", [connector_source_credentials_service_1.ConnectorSourceCredentialsService])
+    __metadata("design:paramtypes", [connector_source_credentials_service_1.ConnectorSourceCredentialsService,
+        connector_definition_service_1.ConnectorDefinitionService])
 ], ConnectorService);
 //# sourceMappingURL=connector.service.js.map

package/dist/src/data-marts/services/connector/declarative-manifest-schemas.d.ts

@@ -0,0 +1,4 @@
+import { ConnectorFieldsSchema } from '../../connector-types/connector-fields-schema';
+import { ConnectorSpecification } from '../../connector-types/connector-specification';
+export declare function specificationFromManifest(manifest: Record<string, unknown>): ConnectorSpecification;
+export declare function fieldsSchemaFromManifest(manifest: Record<string, unknown>): ConnectorFieldsSchema;

package/dist/src/data-marts/services/connector/declarative-manifest-schemas.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.specificationFromManifest = specificationFromManifest;
+exports.fieldsSchemaFromManifest = fieldsSchemaFromManifest;
+const common_1 = require("@nestjs/common");
+const connectors_1 = require("@owox/connectors");
+const connector_fields_schema_1 = require("../../connector-types/connector-fields-schema");
+const connector_specification_1 = require("../../connector-types/connector-specification");
+const connector_fields_schema_mapper_1 = require("./connector-fields-schema.mapper");
+const connector_specification_mapper_1 = require("./connector-specification.mapper");
+function specificationFromManifest(manifest) {
+    const source = declarativeSourceFromManifest(manifest);
+    return parseOrExplain(connector_specification_1.ConnectorSpecification, (0, connector_specification_mapper_1.mapConnectorSpecification)(source.parameters), 'parameter');
+}
+function fieldsSchemaFromManifest(manifest) {
+    const source = declarativeSourceFromManifest(manifest);
+    return parseOrExplain(connector_fields_schema_1.ConnectorFieldsSchema, (0, connector_fields_schema_mapper_1.mapConnectorFieldsSchema)(source.getFieldsSchema()), 'node');
+}
+function declarativeSourceFromManifest(manifest) {
+    const context = new connectors_1.Core.AbstractContext({
+        source: { name: 'custom', config: {} },
+        storage: { name: 'unused', config: {} },
+        runConfig: {},
+        env: { datamartId: null, runId: null },
+    });
+    let model;
+    try {
+        model = new connectors_1.Core.ManifestParser().parse(JSON.stringify(manifest));
+    }
+    catch (e) {
+        throw new common_1.BadRequestException(`Invalid declarative manifest: ${e.message}`);
+    }
+    return new connectors_1.Core.DeclarativeSource(context, model);
+}
+const MANIFEST_KEYS = { title: 'label', required: 'isRequired' };
+function parseOrExplain(schema, items, kind) {
+    const parsed = schema.safeParse(items);
+    if (parsed.success)
+        return parsed.data;
+    const problems = parsed.error.issues.map(issue => {
+        const [index, ...path] = issue.path;
+        const name = typeof index === 'number' ? items[index]?.name : undefined;
+        const where = path.map(key => MANIFEST_KEYS[key] ?? key).join('.');
+        return `${kind} "${String(name)}" ${where}: ${issue.message}`;
+    });
+    throw new common_1.BadRequestException(`Invalid declarative manifest: ${problems.join('; ')}`);
+}
+//# sourceMappingURL=declarative-manifest-schemas.js.map

package/dist/src/data-marts/services/connector/run-log-flusher.d.ts

@@ -0,0 +1,21 @@
+import { ConnectorMessage } from '../../connector-types/connector-message/schemas/connector-message.schema';
+export interface RunLogSnapshot {
+    logs: string[];
+    errors: string[];
+}
+export declare function createRunLogSnapshotReader(logs: ConnectorMessage[], errors: ConnectorMessage[]): () => RunLogSnapshot;
+export declare class RunLogFlusher {
+    private readonly intervalMs;
+    private readonly getSnapshot;
+    private readonly write;
+    private readonly onError?;
+    private timer;
+    private lastCount;
+    private inFlight;
+    private persisted;
+    constructor(intervalMs: number, getSnapshot: () => RunLogSnapshot, write: (snapshot: RunLogSnapshot) => Promise<void>, onError?: ((error: unknown) => void) | undefined);
+    start(): void;
+    flushIfDirty(): Promise<void>;
+    persistedSnapshot(): RunLogSnapshot | null;
+    stop(): Promise<void>;
+}