@owox/backend 0.35.0 → 0.36.0

package/dist/src/data-marts/services/connector/connector-executor.service.js

@@ -13,20 +13,23 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
 };
 var ConnectorExecutorService_1;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ConnectorExecutorService = void 0;
+exports.ConnectorExecutorService = exports.MAX_MERGED_RUN_OUTPUT_ENTRIES = void 0;
 const common_1 = require("@nestjs/common");
+const config_1 = require("@nestjs/config");
 const typeorm_1 = require("@nestjs/typeorm");
 const typeorm_2 = require("typeorm");
 const connectors_1 = require("@owox/connectors");
+const internal_helpers_1 = require("@owox/internal-helpers");
 const { ConfigDto, GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD } = connectors_1.Core;
 const GENERATED_REFRESH_TOKEN_MAX_LENGTH = 4096;
-const MAX_MERGED_RUN_OUTPUT_ENTRIES = 10000;
+exports.MAX_MERGED_RUN_OUTPUT_ENTRIES = 10000;
 const MAX_MERGED_RUN_OUTPUT_BYTES = 6 * 1024 * 1024;
+const MAX_RUN_BUFFER_ENTRIES = exports.MAX_MERGED_RUN_OUTPUT_ENTRIES - 1;
+const MAX_TRACE_ENTRIES = 1000;
 const data_mart_run_entity_1 = require("../../entities/data-mart-run.entity");
 const data_mart_run_status_enum_1 = require("../../enums/data-mart-run-status.enum");
 const project_operation_blocked_exception_1 = require("../../../common/exceptions/project-operation-blocked.exception");
 const connector_output_capture_service_1 = require("../../connector-types/connector-message/services/connector-output-capture.service");
-const internal_helpers_1 = require("@owox/internal-helpers");
 const connector_message_type_enum_1 = require("../../connector-types/enums/connector-message-type-enum");
 const manual_backfill_range_1 = require("../../utils/manual-backfill-range");
 const connector_state_service_1 = require("../../connector-types/connector-message/services/connector-state.service");
@@ -38,6 +41,7 @@ const google_oauth_exceptions_1 = require("../../exceptions/google-oauth.excepti
 const owox_event_dispatcher_1 = require("../../../common/event-dispatcher/owox-event-dispatcher");
 const connector_run_event_1 = require("../../events/connector-run.event");
 const project_billing_service_1 = require("../project-billing/project-billing.service");
+const connector_definition_service_1 = require("./connector-definition.service");
 const connector_process_spawner_service_1 = require("./connector-process-spawner.service");
 const connector_storage_config_service_1 = require("./connector-storage-config.service");
 const connector_source_config_service_1 = require("./connector-source-config.service");
@@ -45,6 +49,10 @@ const connector_credential_injector_service_1 = require("./connector-credential-
 const connector_source_credentials_service_1 = require("./connector-source-credentials.service");
 const connector_message_utils_1 = require("./connector-message.utils");
 const data_mart_run_cancellation_1 = require("../../utils/data-mart-run-cancellation");
+const run_log_flusher_1 = require("./run-log-flusher");
+function addBoundedMessage(array, message) {
+    (0, connector_message_utils_1.addMessageToArray)(array, message, MAX_RUN_BUFFER_ENTRIES);
+}
 let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecutorService {
     dataMartRunRepository;
     processSpawner;
@@ -58,9 +66,11 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
     eventDispatcher;
     projectBillingService;
     dataMartService;
+    connectorDefinitionService;
     connectorSourceCredentialsService;
+    configService;
     logger = new common_1.Logger(ConnectorExecutorService_1.name);
-    constructor(dataMartRunRepository, processSpawner, storageConfigService, sourceConfigService, credentialInjector, connectorOutputCaptureService, connectorStateService, gracefulShutdownService, systemTimeService, eventDispatcher, projectBillingService, dataMartService, connectorSourceCredentialsService) {
+    constructor(dataMartRunRepository, processSpawner, storageConfigService, sourceConfigService, credentialInjector, connectorOutputCaptureService, connectorStateService, gracefulShutdownService, systemTimeService, eventDispatcher, projectBillingService, dataMartService, connectorDefinitionService, connectorSourceCredentialsService, configService) {
         this.dataMartRunRepository = dataMartRunRepository;
         this.processSpawner = processSpawner;
         this.storageConfigService = storageConfigService;
@@ -73,16 +83,23 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
         this.eventDispatcher = eventDispatcher;
         this.projectBillingService = projectBillingService;
         this.dataMartService = dataMartService;
+        this.connectorDefinitionService = connectorDefinitionService;
         this.connectorSourceCredentialsService = connectorSourceCredentialsService;
+        this.configService = configService;
     }
     async executeInBackground(dataMart, run, payload, signal) {
         const runId = run.id;
         const processId = `connector-run-${runId}`;
+        const hasOutputFromEarlierAttempt = (run.logs?.length ?? 0) > 0 || (run.errors?.length ?? 0) > 0;
         this.gracefulShutdownService.registerActiveProcess(processId);
         const capturedLogs = [];
         const capturedErrors = [];
         let configurationResults = [];
-        let hasSuccessfulRun = false;
+        const liveLogs = [];
+        const liveErrors = [];
+        let logFlusher = null;
+        let allConfigurationsSucceeded = false;
+        let hasAnySuccessfulConfiguration = false;
         let wasCancelled = false;
         let operationBlockedException;
         try {
@@ -100,21 +117,24 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                 this.logger.log(`Skipping connector execution for run ${runId}: run reached a terminal status before execution started`, { dataMartId: dataMart.id, projectId: dataMart.projectId, runId });
                 return;
             }
-            configurationResults = await this.runConnectorConfigurations(run, processId, dataMart, payload, signal);
+            logFlusher = this.createRunLogFlusher(runId, liveLogs, liveErrors, hasOutputFromEarlierAttempt);
+            logFlusher?.start();
+            configurationResults = await this.runConnectorConfigurations(run, processId, dataMart, payload, signal, liveLogs, liveErrors);
             configurationResults.forEach(result => {
-                result.logs.forEach(log => (0, connector_message_utils_1.addMessageToArray)(capturedLogs, log));
-                result.errors.forEach(error => (0, connector_message_utils_1.addMessageToArray)(capturedErrors, error));
+                result.logs.forEach(log => addBoundedMessage(capturedLogs, log));
+                result.errors.forEach(error => addBoundedMessage(capturedErrors, error));
             });
             const successCount = configurationResults.filter(r => r.success).length;
             const totalCount = configurationResults.length;
-            hasSuccessfulRun = successCount > 0;
-            wasCancelled = signal?.aborted === true && !hasSuccessfulRun;
+            allConfigurationsSucceeded = totalCount > 0 && successCount === totalCount;
+            hasAnySuccessfulConfiguration = successCount > 0;
+            wasCancelled = signal?.aborted === true && !allConfigurationsSucceeded;
             this.logger.log(`Connector execution completed: ${successCount}/${totalCount} configurations successful`, { dataMartId: dataMart.id, projectId: dataMart.projectId, runId, successCount, totalCount });
         }
         catch (error) {
-            wasCancelled = signal?.aborted === true && !hasSuccessfulRun;
+            wasCancelled = signal?.aborted === true && !allConfigurationsSucceeded;
             const errorMessage = error instanceof Error ? error.message : String(error);
-            (0, connector_message_utils_1.addMessageToArray)(capturedErrors, {
+            addBoundedMessage(capturedErrors, {
                 type: connector_message_type_enum_1.ConnectorMessageType.ERROR,
                 at: this.systemTimeService.now().toISOString(),
                 error: errorMessage,
@@ -142,7 +162,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             catch (error) {
                 const fieldsUpdateError = error instanceof Error ? error.message : String(error);
                 const warning = 'Connector data was imported, but the source field list could not be synchronized. It will be retried on the next run.';
-                (0, connector_message_utils_1.addMessageToArray)(capturedLogs, {
+                addBoundedMessage(capturedLogs, {
                     type: connector_message_type_enum_1.ConnectorMessageType.WARNING,
                     at: this.systemTimeService.now().toISOString(),
                     warning,
@@ -158,12 +178,18 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             if (hasSuccessfulFieldsUpdate) {
                 await this.actualizeSchemaAfterConnectorExecution(dataMart, runId);
             }
-            const statusPersisted = await this.updateRunStatus(runId, hasSuccessfulRun, capturedLogs, capturedErrors, operationBlockedException, wasCancelled);
-            if (hasSuccessfulRun && statusPersisted) {
+            await logFlusher?.stop();
+            const flushedSnapshot = logFlusher?.persistedSnapshot() ?? null;
+            const persistedStatus = await this.updateRunStatus(runId, allConfigurationsSucceeded, capturedLogs, capturedErrors, operationBlockedException, wasCancelled, flushedSnapshot);
+            if (hasAnySuccessfulConfiguration &&
+                persistedStatus !== null &&
+                persistedStatus !== data_mart_run_status_enum_1.DataMartRunStatus.INTERRUPTED) {
                 await this.projectBillingService.registerConnectorRunConsumption(dataMart, runId);
+            }
+            if (allConfigurationsSucceeded && persistedStatus !== null) {
                 await this.eventDispatcher.publishExternal(new connector_run_event_1.ConnectorRunEvent(dataMart.id, runId, dataMart.projectId, run.createdById ?? 'system', run.runType, 'successfully'));
             }
-            else if (statusPersisted &&
+            else if (persistedStatus !== null &&
                 !wasCancelled &&
                 !this.gracefulShutdownService.isInShutdownMode()) {
                 await this.eventDispatcher.publishExternal(new connector_run_event_1.ConnectorRunEvent(dataMart.id, runId, dataMart.projectId, run.createdById ?? 'system', run.runType, 'unsuccessfully'));
@@ -199,7 +225,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
         }
     }
-    async runConnectorConfigurations(run, processId, dataMart, payload, signal) {
+    async runConnectorConfigurations(run, processId, dataMart, payload, signal, liveLogs = [], liveErrors = []) {
         const runId = run.id;
         const definition = dataMart.definition;
         const { connector } = definition;
@@ -207,6 +233,8 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
         const runBody = (0, manual_backfill_range_1.unwrapRunPayload)(payload);
         const isBackfill = (0, manual_backfill_range_1.isManualBackfillPayload)(runBody);
         let backfillProgress = (0, manual_backfill_range_1.readBackfillProgress)(run.additionalParams);
+        const customManifest = await this.connectorDefinitionService.tryResolveManifest(dataMart.projectId, connector.source.name, connector.source.version);
+        const manifestForRunner = customManifest ? this.stripManifestForRunner(customManifest) : null;
         for (const [configIndex, config] of connector.source.configuration.entries()) {
             const configId = config._id;
             if (!configId) {
@@ -234,7 +262,23 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
             const configLogs = [];
             const configErrors = [];
+            let tracesKept = 0;
             let progressWrite = Promise.resolve();
+            let stateWrite = Promise.resolve();
+            const persistState = (state, at) => {
+                stateWrite = stateWrite.then(() => this.connectorStateService
+                    .updateState(dataMart.id, configId, { state, at })
+                    .catch(error => {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    this.logger.error(`Failed to save state: ${errorMessage}`, error?.stack, {
+                        dataMartId: dataMart.id,
+                        projectId: dataMart.projectId,
+                        runId,
+                        configId,
+                        error: errorMessage,
+                    });
+                }));
+            };
             const resume = (0, manual_backfill_range_1.resumeManualBackfillPayload)(runBody, backfillProgress[configId]);
             let success = false;
             let credentialUpdates;
@@ -244,7 +288,8 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             const logCaptureConfig = this.connectorOutputCaptureService.createCapture((message) => {
                 switch (message.type) {
                     case connector_message_type_enum_1.ConnectorMessageType.ERROR:
-                        (0, connector_message_utils_1.addMessageToArray)(configErrors, message);
+                        addBoundedMessage(configErrors, message);
+                        addBoundedMessage(liveErrors, message);
                         this.logger.error(`${message.toFormattedString()}`, {
                             dataMartId: dataMart.id,
                             projectId: dataMart.projectId,
@@ -253,7 +298,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                         });
                         break;
                     case connector_message_type_enum_1.ConnectorMessageType.WARNING:
-                        (0, connector_message_utils_1.addMessageToArray)(configErrors, message);
+                        addBoundedMessage(configErrors, message);
                         this.logger.warn(`${message.toFormattedString()}`, {
                             dataMartId: dataMart.id,
                             projectId: dataMart.projectId,
@@ -261,23 +306,14 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                             configId,
                         });
                         break;
+                    case connector_message_type_enum_1.ConnectorMessageType.STATE_UPDATE:
+                        persistState(message.state, message.at);
+                        break;
                     case connector_message_type_enum_1.ConnectorMessageType.REQUESTED_DATE: {
+                        addBoundedMessage(configLogs, message);
+                        addBoundedMessage(liveLogs, message);
                         if (!isBackfill) {
-                            this.connectorStateService
-                                .updateState(dataMart.id, configId, {
-                                state: { date: message.date },
-                                at: message.at,
-                            })
-                                .catch(error => {
-                                const errorMessage = error instanceof Error ? error.message : String(error);
-                                this.logger.error(`Failed to save state: ${errorMessage}`, error?.stack, {
-                                    dataMartId: dataMart.id,
-                                    projectId: dataMart.projectId,
-                                    runId,
-                                    configId,
-                                    error: errorMessage,
-                                });
-                            });
+                            persistState({ date: message.date }, message.at);
                             break;
                         }
                         const completedDay = (0, manual_backfill_range_1.parseBackfillDay)(message.date);
@@ -311,7 +347,8 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                     case connector_message_type_enum_1.ConnectorMessageType.STATUS:
                         if (message.status === connectors_1.Core.EXECUTION_STATUS.ERROR) {
                             success = false;
-                            (0, connector_message_utils_1.addMessageToArray)(configLogs, message);
+                            addBoundedMessage(configLogs, message);
+                            addBoundedMessage(liveLogs, message);
                             this.logger.warn(`${message.toFormattedString()}`, {
                                 dataMartId: dataMart.id,
                                 projectId: dataMart.projectId,
@@ -321,7 +358,8 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                         }
                         else if (this.isSuccessfulConnectorStatus(message.status)) {
                             success = true;
-                            (0, connector_message_utils_1.addMessageToArray)(configLogs, message);
+                            addBoundedMessage(configLogs, message);
+                            addBoundedMessage(liveLogs, message);
                             this.logger.log(`${message.status}`, {
                                 dataMartId: dataMart.id,
                                 projectId: dataMart.projectId,
@@ -330,7 +368,8 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                             });
                         }
                         else {
-                            (0, connector_message_utils_1.addMessageToArray)(configLogs, message);
+                            addBoundedMessage(configLogs, message);
+                            addBoundedMessage(liveLogs, message);
                             this.logger.log(`${message.status}`, {
                                 dataMartId: dataMart.id,
                                 projectId: dataMart.projectId,
@@ -340,7 +379,23 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                         }
                         break;
                     default:
-                        (0, connector_message_utils_1.addMessageToArray)(configLogs, message);
+                        if (message.type === connector_message_type_enum_1.ConnectorMessageType.LOG &&
+                            message.eventType === 'TRACE' &&
+                            ++tracesKept > MAX_TRACE_ENTRIES) {
+                            if (tracesKept === MAX_TRACE_ENTRIES + 1) {
+                                const notice = {
+                                    type: connector_message_type_enum_1.ConnectorMessageType.LOG,
+                                    at: message.at,
+                                    message: `Only the first ${MAX_TRACE_ENTRIES} request traces of this run are kept.`,
+                                    toFormattedString: () => `[LOG] Only the first ${MAX_TRACE_ENTRIES} request traces of this run are kept.`,
+                                };
+                                addBoundedMessage(configLogs, notice);
+                                addBoundedMessage(liveLogs, notice);
+                            }
+                            break;
+                        }
+                        addBoundedMessage(configLogs, message);
+                        addBoundedMessage(liveLogs, message);
                         this.logger.log(`${message.toFormattedString()}`, {
                             dataMartId: dataMart.id,
                             projectId: dataMart.projectId,
@@ -382,7 +437,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                     });
                 }
                 const runConfig = this.sourceConfigService.buildRunConfig(resume.body, configState);
-                await this.processSpawner.spawnConnector(dataMart.id, runId, configuration, runConfig, logCaptureConfig, signal);
+                await this.processSpawner.spawnConnector(dataMart.id, runId, configuration, runConfig, logCaptureConfig, signal, manifestForRunner);
                 if (success) {
                     this.logger.log(`Configuration ${configIndex + 1} completed successfully`, {
                         dataMartId: dataMart.id,
@@ -404,7 +459,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                         configId,
                         configIndex,
                     };
-                    (0, connector_message_utils_1.addMessageToArray)(configErrors, wasInterrupted
+                    addBoundedMessage(configErrors, wasInterrupted
                         ? {
                             type: connector_message_type_enum_1.ConnectorMessageType.WARNING,
                             at,
@@ -439,7 +494,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                     configIndex,
                     error: errorMessage,
                 };
-                (0, connector_message_utils_1.addMessageToArray)(configErrors, isWarning
+                addBoundedMessage(configErrors, isWarning
                     ? {
                         type: connector_message_type_enum_1.ConnectorMessageType.WARNING,
                         at,
@@ -461,15 +516,26 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
             finally {
                 await progressWrite;
+                await stateWrite;
                 if (credentialUpdates) {
                     try {
-                        await this.saveConnectorCredentials(configForCredentialUpdates, credentialUpdates, expectedCredentialValues, dataMart, runId, configId);
+                        const credentialsPersisted = await this.saveConnectorCredentials(configForCredentialUpdates, credentialUpdates, expectedCredentialValues, dataMart, runId, configId);
+                        if (!credentialsPersisted) {
+                            const warning = 'Connector data was imported, but the refreshed credential could not be saved. ' +
+                                'If the next run fails to authenticate, reconnect this source.';
+                            addBoundedMessage(configLogs, {
+                                type: connector_message_type_enum_1.ConnectorMessageType.WARNING,
+                                at: this.systemTimeService.now().toISOString(),
+                                warning,
+                                toFormattedString: () => `[WARNING] ${warning}`,
+                            });
+                        }
                     }
                     catch (error) {
                         success = false;
                         const errorMessage = error instanceof Error ? error.message : String(error);
                         const credentialErrorMessage = `Failed to update connector credentials: ${errorMessage}`;
-                        (0, connector_message_utils_1.addMessageToArray)(configErrors, {
+                        addBoundedMessage(configErrors, {
                             type: connector_message_type_enum_1.ConnectorMessageType.ERROR,
                             at: this.systemTimeService.now().toISOString(),
                             error: credentialErrorMessage,
@@ -525,25 +591,33 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                 });
             }
             if (Object.keys(credentialUpdates).length === 0) {
-                return;
+                return true;
             }
             const credentialId = this.getCredentialIdForConfig(config);
             if (!credentialId) {
-                this.logger.debug(`Skipping connector credential update: no credential reference found`, {
+                this.logger.warn(`Rotated credential was not persisted: this connector has no stored credential. ` +
+                    `Configure it with a stored credential so the rotated token survives the next run.`, {
                     dataMartId: dataMart.id,
                     projectId: dataMart.projectId,
                     runId,
                     configId,
                     credentialKeys: Object.keys(credentials),
                 });
-                return;
-            }
-            if (expectedCredentialValues) {
-                await this.connectorSourceCredentialsService.updateCredentialFields(credentialId, dataMart.projectId, credentialUpdates, expectedCredentialValues);
+                return false;
             }
-            else {
-                await this.connectorSourceCredentialsService.updateCredentialFields(credentialId, dataMart.projectId, credentialUpdates);
+            const result = expectedCredentialValues
+                ? await this.connectorSourceCredentialsService.updateCredentialFields(credentialId, dataMart.projectId, credentialUpdates, expectedCredentialValues)
+                : await this.connectorSourceCredentialsService.updateCredentialFields(credentialId, dataMart.projectId, credentialUpdates);
+            if (!result.updated) {
+                this.logger.warn(`Rotated connector credential was not persisted`, {
+                    dataMartId: dataMart.id,
+                    projectId: dataMart.projectId,
+                    runId,
+                    configId,
+                    credentialId,
+                });
             }
+            return result.updated;
         }
         catch (error) {
             const errorMessage = error instanceof Error ? error.message : String(error);
@@ -651,20 +725,39 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
     isSuccessfulConnectorStatus(status) {
         return status === connectors_1.Core.EXECUTION_STATUS.IMPORT_DONE;
     }
-    async updateRunStatus(runId, hasSuccessfulRun, capturedLogs, capturedErrors, operationBlockedException, wasCancelled = false) {
+    stripManifestForRunner(manifest) {
+        const { logo: _logo, ...rest } = manifest;
+        return rest;
+    }
+    createRunLogFlusher(runId, liveLogs, liveErrors, hasOutputFromEarlierAttempt) {
+        if (hasOutputFromEarlierAttempt)
+            return null;
+        const intervalMs = this.configService.get('CONNECTOR_RUN_LOG_FLUSH_INTERVAL_MS', 2000);
+        if (intervalMs <= 0)
+            return null;
+        return new run_log_flusher_1.RunLogFlusher(intervalMs, (0, run_log_flusher_1.createRunLogSnapshotReader)(liveLogs, liveErrors), async ({ logs, errors }) => {
+            await this.dataMartRunRepository.update(runId, {
+                logs: logs.length > 0 ? logs : null,
+                errors: errors.length > 0 ? errors : null,
+            });
+        }, error => this.logger.warn(`Incremental log flush failed for run ${runId}: ${(0, internal_helpers_1.castError)(error).message}`));
+    }
+    async updateRunStatus(runId, allConfigurationsSucceeded, capturedLogs, capturedErrors, operationBlockedException, wasCancelled = false, flushedSnapshot = null) {
         let status = wasCancelled
             ? data_mart_run_status_enum_1.DataMartRunStatus.CANCELLED
-            : hasSuccessfulRun
+            : allConfigurationsSucceeded
                 ? data_mart_run_status_enum_1.DataMartRunStatus.SUCCESS
                 : operationBlockedException
                     ? data_mart_run_status_enum_1.DataMartRunStatus.RESTRICTED
                     : data_mart_run_status_enum_1.DataMartRunStatus.FAILED;
-        if (!wasCancelled && !hasSuccessfulRun && this.gracefulShutdownService.isInShutdownMode()) {
+        if (!wasCancelled &&
+            !allConfigurationsSucceeded &&
+            this.gracefulShutdownService.isInShutdownMode()) {
             status = data_mart_run_status_enum_1.DataMartRunStatus.INTERRUPTED;
         }
         const newLogStrings = capturedLogs.map(log => JSON.stringify(log));
         const newErrorStrings = capturedErrors.map(error => JSON.stringify(error));
-        const { logs: logsToSave, errors: errorsToSave } = await this.mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings);
+        const { logs: logsToSave, errors: errorsToSave } = await this.mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings, flushedSnapshot);
         const result = await this.dataMartRunRepository.update({ id: runId, status: (0, typeorm_2.In)(data_mart_run_cancellation_1.NON_TERMINAL_DATA_MART_RUN_STATUSES) }, {
             status,
             finishedAt: this.systemTimeService.now(),
@@ -672,18 +765,18 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             errors: errorsToSave,
         });
         if (result.affected) {
-            return true;
+            return status;
         }
         this.logger.log(`Skipped final status update for run ${runId}: run already reached a terminal status`);
         if (newLogStrings.length > 0 || newErrorStrings.length > 0) {
             await this.dataMartRunRepository.update({ id: runId }, { logs: logsToSave, errors: errorsToSave });
         }
-        return false;
+        return null;
     }
-    async mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings) {
+    async mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings, flushedSnapshot = null) {
         const existing = await this.dataMartRunRepository.findOne({ where: { id: runId } });
-        const existingLogs = existing?.logs ?? [];
-        const existingErrors = existing?.errors ?? [];
+        const existingLogs = this.discountFlushedEntries(existing?.logs ?? [], flushedSnapshot?.logs, newLogStrings);
+        const existingErrors = this.discountFlushedEntries(existing?.errors ?? [], flushedSnapshot?.errors, newErrorStrings);
         const mergedLogs = this.capMergedEntries([...existingLogs, ...newLogStrings]);
         const mergedErrors = this.capMergedEntries([...existingErrors, ...newErrorStrings]);
         return {
@@ -691,10 +784,25 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             errors: mergedErrors.length > 0 ? mergedErrors : null,
         };
     }
+    discountFlushedEntries(persisted, flushed, replacement) {
+        if (!flushed?.length || flushed.length > persisted.length) {
+            return persisted;
+        }
+        if (flushed.length > replacement.length) {
+            return persisted;
+        }
+        const start = persisted.length - flushed.length;
+        for (let i = 0; i < flushed.length; i++) {
+            if (persisted[start + i] !== flushed[i]) {
+                return persisted;
+            }
+        }
+        return persisted.slice(0, start);
+    }
     capMergedEntries(entries) {
         let keptBytes = 0;
         let keep = 0;
-        while (keep < entries.length && keep < MAX_MERGED_RUN_OUTPUT_ENTRIES) {
+        while (keep < entries.length && keep < exports.MAX_MERGED_RUN_OUTPUT_ENTRIES) {
             const entryBytes = Buffer.byteLength(JSON.stringify(entries[entries.length - 1 - keep])) + 1;
             if (keptBytes + entryBytes > MAX_MERGED_RUN_OUTPUT_BYTES) {
                 break;
@@ -705,7 +813,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
         if (keep === entries.length) {
             return entries;
         }
-        keep = Math.min(keep, MAX_MERGED_RUN_OUTPUT_ENTRIES - 1);
+        keep = Math.min(keep, exports.MAX_MERGED_RUN_OUTPUT_ENTRIES - 1);
         const dropped = entries.length - keep;
         const truncationNotice = JSON.stringify({
             type: connector_message_type_enum_1.ConnectorMessageType.LOG,
@@ -731,6 +839,8 @@ exports.ConnectorExecutorService = ConnectorExecutorService = ConnectorExecutorS
         owox_event_dispatcher_1.OwoxEventDispatcher,
         project_billing_service_1.ProjectBillingService,
         data_mart_service_1.DataMartService,
-        connector_source_credentials_service_1.ConnectorSourceCredentialsService])
+        connector_definition_service_1.ConnectorDefinitionService,
+        connector_source_credentials_service_1.ConnectorSourceCredentialsService,
+        config_1.ConfigService])
 ], ConnectorExecutorService);
 //# sourceMappingURL=connector-executor.service.js.map

package/dist/src/data-marts/services/connector/connector-fields-preview.service.js

@@ -50,7 +50,7 @@ let ConnectorFieldsPreviewService = ConnectorFieldsPreviewService_1 = class Conn
             throw (0, connector_preview_support_1.mapConnectorPreviewError)(error, this.logger, PREVIEW_ERROR_MESSAGES);
         }
         try {
-            source.config.validate();
+            source.context.validate();
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);

package/dist/src/data-marts/services/connector/connector-message.utils.js

@@ -43,9 +43,9 @@ function addMessageToArray(array, message, maxCount) {
         if (array.length === maxCount) {
             logger.warn(`Maximum number of messages (${maxCount}) reached.`);
             array.push({
-                type: connector_message_type_enum_1.ConnectorMessageType.ERROR,
+                type: connector_message_type_enum_1.ConnectorMessageType.WARNING,
                 at: new Date().toISOString(),
-                error: `Maximum number of messages (${maxCount}) reached.`,
+                warning: `Maximum number of messages (${maxCount}) reached.`,
                 toFormattedString: () => `[WARNING] Maximum number of messages (${maxCount}) reached.`,
             });
         }

package/dist/src/data-marts/services/connector/connector-preview-credentials.service.js

@@ -27,7 +27,7 @@ let ConnectorPreviewCredentialsService = class ConnectorPreviewCredentialsServic
     async inject(connectorName, config, context) {
         const previewConfig = this.withoutStoredSecretReferenceForInlineKey(config);
         await this.validateReferences(connectorName, previewConfig, context);
-        const configWithSecrets = await this.credentialInjector.injectSecrets(previewConfig, context.projectId);
+        const configWithSecrets = await this.credentialInjector.injectSecrets(previewConfig, context.projectId, connectorName);
         return this.credentialInjector.injectOAuthCredentials(configWithSecrets, connectorName, context.projectId);
     }
     withoutStoredSecretReferenceForInlineKey(config) {

package/dist/src/data-marts/services/connector/connector-preview-support.d.ts

@@ -5,7 +5,7 @@ export interface ConnectorPreviewErrorMessages {
     unexpectedLog: string;
 }
 export interface ConnectorPreviewSource {
-    config: {
+    context: {
         validate(): void;
     };
     fetchFieldsSchema(signal: AbortSignal): Promise<unknown>;

package/dist/src/data-marts/services/connector/connector-preview-support.js

@@ -9,22 +9,12 @@ const connectors_1 = require("@owox/connectors");
 const CONNECTOR_PREVIEW_TIMEOUT_MS = 15_000;
 class ConnectorPreviewTimeoutError extends Error {
 }
-class ConnectorPreviewConfig extends connectors_1.Core.AbstractConfig {
-    logger;
-    constructor(configData, logger) {
-        super(configData);
-        this.logger = logger;
-    }
-    handleStatusUpdate() { }
-    updateLastImportDate() { }
-    updateLastRequstedDate() { }
-    isInProgress() {
-        return false;
-    }
-    addWarningToCurrentStatus() { }
-    logMessage(message) {
-        this.logger.debug(message);
-    }
+function createConnectorPreviewContext(init, logger) {
+    const context = new connectors_1.Core.AbstractContext(init);
+    context.emit = (event) => {
+        logger?.debug(JSON.stringify(event.toJSON()));
+    };
+    return context;
 }
 function getConnectorSourceClass(connectorName) {
     return connectors_1.Connectors[connectorName]?.[`${connectorName}Source`];
@@ -35,11 +25,14 @@ function connectorSourceImplements(connectorName, method) {
 }
 function createConnectorPreviewSource(connectorName, configuration, logger) {
     const SourceClass = connectors_1.Connectors[connectorName][`${connectorName}Source`];
-    const sourceConfig = new connectors_1.Core.SourceConfigDto({
-        name: connectorName,
-        config: configuration,
-    });
-    return new SourceClass(new ConnectorPreviewConfig(sourceConfig.config, logger));
+    const { config } = new connectors_1.Core.SourceConfigDto({ name: connectorName, config: configuration });
+    const context = createConnectorPreviewContext({
+        source: { name: connectorName, config },
+        storage: { name: 'unused', config: {} },
+        runConfig: {},
+        env: { datamartId: null, runId: null },
+    }, logger);
+    return new SourceClass(context);
 }
 async function withConnectorPreviewTimeout(work, timeoutMessage) {
     const abortController = new AbortController();

package/dist/src/data-marts/services/connector/connector-process-spawner.service.d.ts

@@ -2,6 +2,15 @@ import { GracefulShutdownService } from '../../../common/scheduler/services/grac
 import { Core } from '@owox/connectors';
 type ConfigDto = InstanceType<typeof Core.ConfigDto>;
 type RunConfigDto = InstanceType<typeof Core.RunConfigDto>;
+export declare const MAX_CAPTURED_LINE_LENGTH: number;
+export declare const TRUNCATED_OUTPUT_LINE = "[TRUNCATED connector output line: exceeded 1048576 bytes]";
+export declare const INHERITED_CONNECTOR_ENV_VARS: readonly ["PATH", "NODE_OPTIONS", "NODE_EXTRA_CA_CERTS", "SSL_CERT_FILE", "SSL_CERT_DIR", "HOME", "USERPROFILE", "TMPDIR", "TMP", "TEMP", "SystemRoot"];
+export declare function inheritConnectorEnv(names?: readonly string[]): Record<string, string | undefined>;
+export interface CapturedLineBuffer {
+    push: (chunk: string) => void;
+    flush: () => void;
+}
+export declare function createCapturedLineBuffer(onLine: (message: string) => void): CapturedLineBuffer;
 export declare class ConnectorProcessSpawnerService {
     private readonly gracefulShutdownService;
     private readonly logger;
@@ -12,7 +21,7 @@ export declare class ConnectorProcessSpawnerService {
             onStderr?: (message: string) => void;
         };
         onSpawn?: (pid: number | undefined) => void;
-    }, signal?: AbortSignal): Promise<void>;
-    private createLineBuffer;
+    }, signal?: AbortSignal, manifest?: Record<string, unknown> | null): Promise<void>;
+    protected buildChildEnv(datamartId: string, runId: string, configuration: ConfigDto, runConfig: RunConfigDto, manifest?: Record<string, unknown> | null): Record<string, string | undefined>;
 }
 export {};