@owox/backend 0.38.0 → 0.39.0

package/dist/src/data-marts/credentials/use-cases/create-credential.service.js

@@ -26,6 +26,7 @@ const credential_context_entity_1 = require("../entities/credential-context.enti
 const credential_owner_entity_1 = require("../entities/credential-owner.entity");
 const credential_definition_service_1 = require("../services/credential-definition.service");
 const credential_service_1 = require("../services/credential.service");
+const google_service_account_key_1 = require("../services/google-service-account-key");
 const credential_view_service_1 = require("../services/credential-view.service");
 const credential_validation_probe_service_1 = require("../services/credential-validation-probe.service");
 const credential_types_1 = require("../credential.types");
@@ -51,10 +52,16 @@ let CreateCredentialService = class CreateCredentialService {
     async run(context, input) {
         const definition = await this.definitions.get(input.definitionId);
         const aiConfiguration = normalizeAiConfiguration(input.aiModelMappings, input.aiModelMappingModes, definition.contract.ai);
+        const publicMetadata = definition.contract.auth.type === 'google-service-account'
+            ? (0, google_service_account_key_1.parseGoogleServiceAccountKey)(input.secret.value).metadata
+            : null;
         const validation = await this.validationProbe.run(definition, { value: input.secret.value });
         if (validation.state === 'rejected') {
             throw new common_1.BadRequestException(validation.message);
         }
+        return this.persist(context, input, definition, aiConfiguration, publicMetadata, validation);
+    }
+    async persist(context, input, definition, aiConfiguration, publicMetadata, validation) {
         const credential = await this.credentials.save(this.credentials.create({
             projectId: context.projectId,
             title: input.title.trim(),
@@ -71,6 +78,7 @@ let CreateCredentialService = class CreateCredentialService {
             validationState: validation.state,
             validationMessage: validation.message,
             validatedAt: validation.validatedAt,
+            publicMetadata,
             createdById: context.userId,
         }));
         const ownerIds = [...new Set([context.userId, ...(input.ownerIds ?? [])])];
@@ -90,9 +98,9 @@ exports.CreateCredentialService = CreateCredentialService;
 __decorate([
     (0, typeorm_transactional_1.Transactional)(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Object, Function]),
+    __metadata("design:paramtypes", [Object, Function, Object, void 0, Object, Object]),
     __metadata("design:returntype", Promise)
-], CreateCredentialService.prototype, "run", null);
+], CreateCredentialService.prototype, "persist", null);
 exports.CreateCredentialService = CreateCredentialService = __decorate([
     (0, common_1.Injectable)(),
     __param(6, (0, typeorm_1.InjectRepository)(credential_owner_entity_1.CredentialOwner)),

package/dist/src/data-marts/credentials/use-cases/update-credential.service.d.ts

@@ -21,5 +21,6 @@ export declare class UpdateCredentialService {
     private readonly owners;
     constructor(credentials: CredentialService, definitions: CredentialDefinitionService, access: AccessDecisionService, contextAccess: ContextAccessService, view: CredentialViewService, validationProbe: CredentialValidationProbeService, idpProjections: IdpProjectionsFacade, owners: Repository<CredentialOwner>);
     run(id: string, context: AuthorizationContext, input: UpdateCredentialApiDto): Promise<CredentialDto>;
+    private apply;
     private assertAccess;
 }

package/dist/src/data-marts/credentials/use-cases/update-credential.service.js

@@ -24,6 +24,7 @@ const sync_owners_1 = require("../../utils/sync-owners");
 const credential_owner_entity_1 = require("../entities/credential-owner.entity");
 const credential_definition_service_1 = require("../services/credential-definition.service");
 const credential_service_1 = require("../services/credential.service");
+const google_service_account_key_1 = require("../services/google-service-account-key");
 const credential_view_service_1 = require("../services/credential-view.service");
 const credential_validation_probe_service_1 = require("../services/credential-validation-probe.service");
 const create_credential_service_1 = require("./create-credential.service");
@@ -65,14 +66,29 @@ let UpdateCredentialService = class UpdateCredentialService {
             ? await this.definitions.getForCredential(credential)
             : await this.definitions.getForView(credential);
         let validation;
-        if (input.title !== undefined)
-            credential.title = input.title.trim();
+        let publicMetadata = credential.publicMetadata;
         if (input.secret !== undefined) {
+            if (definition.contract.auth.type === 'google-service-account') {
+                publicMetadata = (0, google_service_account_key_1.parseGoogleServiceAccountKey)(input.secret.value).metadata;
+                const savedEmail = credential.publicMetadata?.clientEmail ?? storedClientEmail(credential.secret.value);
+                if (savedEmail && publicMetadata.clientEmail.toLowerCase() !== savedEmail.toLowerCase()) {
+                    throw new common_1.BadRequestException(`The replacement key must belong to ${savedEmail}. Create another Credential for a different service account.`);
+                }
+            }
             validation = await this.validationProbe.run(definition, { value: input.secret.value });
             if (validation.state === 'rejected') {
                 throw new common_1.BadRequestException(validation.message);
             }
+        }
+        return this.apply(id, context, input, definition, validation, publicMetadata);
+    }
+    async apply(id, context, input, definition, validation, publicMetadata) {
+        const credential = await this.credentials.getByIdAndProjectId(id, context.projectId);
+        if (input.title !== undefined)
+            credential.title = input.title.trim();
+        if (input.secret !== undefined && validation) {
             credential.secret = { value: input.secret.value };
+            credential.publicMetadata = publicMetadata;
             credential.validationState = validation.state;
             credential.validationMessage = validation.message;
             credential.validatedAt = validation.validatedAt;
@@ -113,9 +129,9 @@ exports.UpdateCredentialService = UpdateCredentialService;
 __decorate([
     (0, typeorm_transactional_1.Transactional)(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, Object, Function]),
+    __metadata("design:paramtypes", [String, Object, Function, Object, Object, Object]),
     __metadata("design:returntype", Promise)
-], UpdateCredentialService.prototype, "run", null);
+], UpdateCredentialService.prototype, "apply", null);
 exports.UpdateCredentialService = UpdateCredentialService = __decorate([
     (0, common_1.Injectable)(),
     __param(7, (0, typeorm_1.InjectRepository)(credential_owner_entity_1.CredentialOwner)),
@@ -128,4 +144,12 @@ exports.UpdateCredentialService = UpdateCredentialService = __decorate([
         idp_projections_facade_1.IdpProjectionsFacade,
         typeorm_2.Repository])
 ], UpdateCredentialService);
+function storedClientEmail(secretValue) {
+    try {
+        return (0, google_service_account_key_1.parseGoogleServiceAccountKey)(secretValue).metadata.clientEmail;
+    }
+    catch {
+        return undefined;
+    }
+}
 //# sourceMappingURL=update-credential.service.js.map

package/dist/src/data-marts/credentials/use-cases/validate-credential.service.js

@@ -36,11 +36,8 @@ let ValidateCredentialService = class ValidateCredentialService {
         }
         const definition = await this.definitions.getForCredential(credential);
         const validation = await this.validationProbe.run(definition, credential.secret);
-        credential.validationState = validation.state;
-        credential.validationMessage = validation.message;
-        credential.validatedAt = validation.validatedAt;
-        await this.credentials.save(credential);
-        return this.view.build(credential);
+        await this.credentials.writeValidation(id, credential.secret, validation);
+        return this.view.build(await this.credentials.getByIdAndProjectId(id, context.projectId));
     }
 };
 exports.ValidateCredentialService = ValidateCredentialService;

package/dist/src/data-marts/data-marts.module.js

@@ -265,6 +265,7 @@ const update_insight_template_source_service_1 = require("./use-cases/update-ins
 const delete_insight_template_source_service_1 = require("./use-cases/delete-insight-template-source.service");
 const delete_insight_template_service_1 = require("./use-cases/delete-insight-template.service");
 const retry_interrupted_connector_runs_processor_1 = require("./system-triggers/processors/retry-interrupted-connector-runs-processor");
+const repair_backfill_chains_processor_1 = require("./system-triggers/processors/repair-backfill-chains-processor");
 const sql_run_service_1 = require("./use-cases/sql-run.service");
 const create_view_service_1 = require("./use-cases/create-view.service");
 const ai_insights_providers_1 = require("./ai-insights/ai-insights-providers");
@@ -477,6 +478,7 @@ const validate_credential_service_1 = require("./credentials/use-cases/validate-
 const credential_consumer_binding_facade_1 = require("./credentials/facades/credential-consumer-binding.facade");
 const credential_consumer_binding_facade_impl_1 = require("./credentials/facades/credential-consumer-binding.facade.impl");
 const credential_fetch_service_1 = require("./credentials/services/credential-fetch.service");
+const google_service_account_token_service_1 = require("./credentials/services/google-service-account-token.service");
 const credential_ai_service_1 = require("./credentials/services/credential-ai.service");
 const credential_validation_probe_service_1 = require("./credentials/services/credential-validation-probe.service");
 const credential_external_definition_registry_service_1 = require("./credentials/services/credential-external-definition-registry.service");
@@ -818,6 +820,7 @@ exports.DataMartsModule = DataMartsModule = __decorate([
             ai_helper_trigger_service_1.AiHelperTriggerService,
             ai_helper_trigger_handler_service_1.AiHelperTriggerHandlerService,
             retry_interrupted_connector_runs_processor_1.RetryInterruptedConnectorRunsProcessor,
+            repair_backfill_chains_processor_1.RepairBackfillChainsProcessor,
             scheduled_trigger_service_1.ScheduledTriggerService,
             scheduled_trigger_mapper_1.ScheduledTriggerMapper,
             create_scheduled_trigger_service_1.CreateScheduledTriggerService,
@@ -994,6 +997,7 @@ exports.DataMartsModule = DataMartsModule = __decorate([
             delete_credential_service_1.DeleteCredentialService,
             validate_credential_service_1.ValidateCredentialService,
             credential_fetch_service_1.CredentialFetchService,
+            google_service_account_token_service_1.GoogleServiceAccountTokenService,
             credential_ai_service_1.CredentialAiService,
             credential_validation_probe_service_1.CredentialValidationProbeService,
             credential_external_definition_registry_service_1.CredentialExternalDefinitionRegistryService,

package/dist/src/data-marts/data-storage-types/snowflake/adapters/snowflake-api.adapter.js

@@ -51,6 +51,7 @@ class SnowflakeApiAdapter {
             warehouse: config.warehouse,
             retryTimeout: 10000,
             sfRetryMaxLoginRetries: 1,
+            certRevocationCheckMode: 'ADVISORY',
         };
         if (credentials.authMethod === snowflake_auth_method_enum_1.SnowflakeAuthMethod.PASSWORD) {
             connectionOptions.username = credentials.username;

package/dist/src/data-marts/dto/presentation/run-data-mart-request-api.dto.js

@@ -171,8 +171,10 @@ __decorate([
         description: `Payload for the manual run. Omit it or select INCREMENTAL for an incremental run.
     MANUAL_BACKFILL can include connector-specific fields in data; connectors without backfill
     fields can omit data. StartDate and EndDate accept YYYY-MM-DD (or the date part of an
-    ISO-8601 timestamp). A missing or future EndDate counts as today, and the resulting range
-    may cover at most 31 days.`,
+    ISO-8601 timestamp). A missing or future EndDate counts as today. A StartDate earlier than
+    2000-01-01 is rejected. The range has no other length limit: a range longer than 31 days runs
+    as a chain of runs, one segment of up to 31 days each. A segment counts as one connector run
+    once any of its accounts loads data.`,
     }),
     __metadata("design:type", Object)
 ], RunDataMartRequestApiDto.prototype, "payload", void 0);

package/dist/src/data-marts/services/connector/backfill-segment.d.ts

@@ -0,0 +1,22 @@
+import { Repository } from 'typeorm';
+import { ConnectorMessage } from '../../connector-types/connector-message/schemas/connector-message.schema';
+import { ConnectorMessageType } from '../../connector-types/enums/connector-message-type-enum';
+import { DataMart } from '../../entities/data-mart.entity';
+import { DataMartRun } from '../../entities/data-mart-run.entity';
+import { BackfillChainMeta, BackfillDateRange } from '../../utils/manual-backfill-range';
+import { ConnectorRunTriggerService } from './connector-run-trigger.service';
+export declare const BACKFILL_SEGMENT_METRIC = "backfill_segment";
+export type SegmentMarkerPhase = 'started' | 'completed' | 'failed';
+export type SegmentChainEnd = 'finished' | 'stopped';
+export interface SegmentMarkerOutcome {
+    readonly loadedThrough?: string;
+    readonly configurationsLoaded?: number;
+    readonly configurationsTotal?: number;
+    readonly counted?: boolean;
+    readonly chainContinues?: boolean;
+    readonly chainEnd?: SegmentChainEnd;
+}
+export declare function buildSegmentMarker(meta: BackfillChainMeta, window: BackfillDateRange, phase: SegmentMarkerPhase, at: string, outcome?: SegmentMarkerOutcome): Extract<ConnectorMessage, {
+    type: ConnectorMessageType.LOG;
+}>;
+export declare function createNextSegmentRun(runRepository: Repository<DataMartRun>, triggerService: ConnectorRunTriggerService, previousRun: DataMartRun, dataMart: DataMart, meta: BackfillChainMeta, window: BackfillDateRange): Promise<DataMartRun>;

package/dist/src/data-marts/services/connector/backfill-segment.js

@@ -0,0 +1,91 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BACKFILL_SEGMENT_METRIC = void 0;
+exports.buildSegmentMarker = buildSegmentMarker;
+exports.createNextSegmentRun = createNextSegmentRun;
+const connector_message_type_enum_1 = require("../../connector-types/enums/connector-message-type-enum");
+const data_mart_run_status_enum_1 = require("../../enums/data-mart-run-status.enum");
+const data_mart_run_type_enum_1 = require("../../enums/data-mart-run-type.enum");
+const manual_backfill_range_1 = require("../../utils/manual-backfill-range");
+exports.BACKFILL_SEGMENT_METRIC = 'backfill_segment';
+function describeLoadedDays(outcome) {
+    if (!outcome.loadedThrough)
+        return 'no days of this segment were loaded';
+    const { configurationsLoaded, configurationsTotal } = outcome;
+    const isPartial = configurationsLoaded !== undefined &&
+        configurationsTotal !== undefined &&
+        configurationsLoaded < configurationsTotal;
+    const scope = isPartial
+        ? ` (${configurationsLoaded} of ${configurationsTotal} configurations)`
+        : '';
+    return `days through ${outcome.loadedThrough} were loaded${scope}`;
+}
+function describeUnloadedPeriod(meta, window) {
+    const isLastSegment = meta.segmentIndex >= meta.segmentsTotal;
+    const segments = isLastSegment
+        ? `segment ${meta.segmentIndex}`
+        : `segments ${meta.segmentIndex}–${meta.segmentsTotal}`;
+    return `${segments} (${window.startDate} – ${meta.requestedEnd}) ${isLastSegment ? 'was' : 'were'} not loaded`;
+}
+function buildSegmentMarker(meta, window, phase, at, outcome = {}) {
+    const period = `${meta.requestedStart} – ${meta.requestedEnd}`;
+    const position = `segment ${meta.segmentIndex} of ${meta.segmentsTotal} (${window.startDate} – ${window.endDate})`;
+    let message;
+    switch (phase) {
+        case 'started':
+            message = `Backfill ${period}: loading ${position}`;
+            break;
+        case 'completed':
+            message = `Backfill ${period}: ${position} completed — counted as 1 run`;
+            break;
+        case 'failed':
+            message =
+                outcome.chainEnd === 'stopped'
+                    ? `Backfill stopped: the project is restricted; ${describeUnloadedPeriod(meta, window)}.`
+                    : `Backfill ${period}: ${position} failed; ${describeLoadedDays(outcome)}${outcome.counted ? ' — counted as 1 run' : ''}`;
+            break;
+    }
+    return {
+        type: connector_message_type_enum_1.ConnectorMessageType.LOG,
+        at,
+        metric: exports.BACKFILL_SEGMENT_METRIC,
+        value: meta.segmentIndex,
+        tags: {
+            phase,
+            segmentStart: window.startDate,
+            segmentEnd: window.endDate,
+            backfillStart: meta.requestedStart,
+            backfillEnd: meta.requestedEnd,
+            segmentsTotal: meta.segmentsTotal,
+            ...(outcome.loadedThrough ? { loadedThrough: outcome.loadedThrough } : {}),
+            ...(outcome.chainContinues === undefined ? {} : { chainContinues: outcome.chainContinues }),
+            ...(outcome.chainEnd ? { chainEnd: outcome.chainEnd } : {}),
+        },
+        message,
+        toFormattedString: () => `[LOG] ${message}`,
+    };
+}
+async function createNextSegmentRun(runRepository, triggerService, previousRun, dataMart, meta, window) {
+    const additionalParams = (0, manual_backfill_range_1.nextSegmentAdditionalParams)(previousRun.additionalParams, meta, window);
+    const nextRun = await runRepository.save(runRepository.create({
+        dataMartId: previousRun.dataMartId,
+        type: data_mart_run_type_enum_1.DataMartRunType.CONNECTOR,
+        definitionRun: dataMart.definition,
+        status: data_mart_run_status_enum_1.DataMartRunStatus.PENDING,
+        createdById: previousRun.createdById ?? undefined,
+        runType: previousRun.runType,
+        logs: [],
+        errors: [],
+        additionalParams,
+    }));
+    await triggerService.createTrigger({
+        dataMartId: previousRun.dataMartId,
+        projectId: dataMart.projectId,
+        createdById: previousRun.createdById ?? 'system',
+        dataMartRunId: nextRun.id,
+        runType: previousRun.runType,
+        payload: additionalParams,
+    });
+    return nextRun;
+}
+//# sourceMappingURL=backfill-segment.js.map

package/dist/src/data-marts/services/connector/connector-execution.service.d.ts

@@ -9,5 +9,6 @@ export declare class ConnectorExecutionService {
     run(dataMart: DataMart, createdById: string, runType: RunType, payload?: Record<string, unknown>): Promise<string>;
     executeExistingRun(dataMart: DataMart, run: DataMartRun, payload?: Record<string, unknown> | null, signal?: AbortSignal): Promise<void>;
     executeInterruptedRuns(): Promise<void>;
+    repairBackfillChains(): Promise<void>;
     getDataMartConnectorRunsByStatus(status: DataMartRunStatus): Promise<DataMartRun[]>;
 }

package/dist/src/data-marts/services/connector/connector-execution.service.js

@@ -26,6 +26,9 @@ let ConnectorExecutionService = class ConnectorExecutionService {
     async executeInterruptedRuns() {
         return this.connectorRunService.executeInterruptedRuns();
     }
+    async repairBackfillChains() {
+        return this.connectorRunService.repairBackfillChains();
+    }
     async getDataMartConnectorRunsByStatus(status) {
         return this.connectorRunService.getDataMartConnectorRunsByStatus(status);
     }

package/dist/src/data-marts/services/connector/connector-executor.service.d.ts

@@ -11,6 +11,7 @@ import { SystemTimeService } from '../../../common/scheduler/services/system-tim
 import { OwoxEventDispatcher } from '../../../common/event-dispatcher/owox-event-dispatcher';
 import { ProjectBillingService } from '../project-billing/project-billing.service';
 import { ConnectorDefinitionService } from './connector-definition.service';
+import { ConnectorRunTriggerService } from './connector-run-trigger.service';
 import { ConnectorProcessSpawnerService } from './connector-process-spawner.service';
 import { ConnectorStorageConfigService } from './connector-storage-config.service';
 import { ConnectorSourceConfigService } from './connector-source-config.service';
@@ -32,8 +33,9 @@ export declare class ConnectorExecutorService {
     private readonly connectorDefinitionService;
     private readonly connectorSourceCredentialsService;
     private readonly configService;
+    private readonly connectorRunTriggerService;
     private readonly logger;
-    constructor(dataMartRunRepository: Repository<DataMartRun>, processSpawner: ConnectorProcessSpawnerService, storageConfigService: ConnectorStorageConfigService, sourceConfigService: ConnectorSourceConfigService, credentialInjector: ConnectorCredentialInjectorService, connectorOutputCaptureService: ConnectorOutputCaptureService, connectorStateService: ConnectorStateService, gracefulShutdownService: GracefulShutdownService, systemTimeService: SystemTimeService, eventDispatcher: OwoxEventDispatcher, projectBillingService: ProjectBillingService, dataMartService: DataMartService, connectorDefinitionService: ConnectorDefinitionService, connectorSourceCredentialsService: ConnectorSourceCredentialsService, configService: ConfigService);
+    constructor(dataMartRunRepository: Repository<DataMartRun>, processSpawner: ConnectorProcessSpawnerService, storageConfigService: ConnectorStorageConfigService, sourceConfigService: ConnectorSourceConfigService, credentialInjector: ConnectorCredentialInjectorService, connectorOutputCaptureService: ConnectorOutputCaptureService, connectorStateService: ConnectorStateService, gracefulShutdownService: GracefulShutdownService, systemTimeService: SystemTimeService, eventDispatcher: OwoxEventDispatcher, projectBillingService: ProjectBillingService, dataMartService: DataMartService, connectorDefinitionService: ConnectorDefinitionService, connectorSourceCredentialsService: ConnectorSourceCredentialsService, configService: ConfigService, connectorRunTriggerService: ConnectorRunTriggerService);
     executeInBackground(dataMart: DataMart, run: DataMartRun, payload?: Record<string, unknown> | null, signal?: AbortSignal): Promise<void>;
     private actualizeSchemaAfterConnectorExecution;
     private runConnectorConfigurations;
@@ -49,6 +51,7 @@ export declare class ConnectorExecutorService {
     private stripManifestForRunner;
     private createRunLogFlusher;
     private updateRunStatus;
+    private writeRunOutcome;
     private mergeWithPersistedOutput;
     private discountFlushedEntries;
     private capMergedEntries;

package/dist/src/data-marts/services/connector/connector-executor.service.js

@@ -18,13 +18,14 @@ const common_1 = require("@nestjs/common");
 const config_1 = require("@nestjs/config");
 const typeorm_1 = require("@nestjs/typeorm");
 const typeorm_2 = require("typeorm");
+const typeorm_transactional_1 = require("typeorm-transactional");
 const connectors_1 = require("@owox/connectors");
 const internal_helpers_1 = require("@owox/internal-helpers");
 const { ConfigDto, GENERATED_REFRESH_TOKEN_CREDENTIAL_FIELD } = connectors_1.Core;
 const GENERATED_REFRESH_TOKEN_MAX_LENGTH = 4096;
 exports.MAX_MERGED_RUN_OUTPUT_ENTRIES = 10000;
 const MAX_MERGED_RUN_OUTPUT_BYTES = 6 * 1024 * 1024;
-const MAX_RUN_BUFFER_ENTRIES = exports.MAX_MERGED_RUN_OUTPUT_ENTRIES - 1;
+const MAX_RUN_BUFFER_ENTRIES = exports.MAX_MERGED_RUN_OUTPUT_ENTRIES - 2;
 const MAX_TRACE_ENTRIES = 1000;
 const data_mart_run_entity_1 = require("../../entities/data-mart-run.entity");
 const data_mart_run_status_enum_1 = require("../../enums/data-mart-run-status.enum");
@@ -42,6 +43,8 @@ const owox_event_dispatcher_1 = require("../../../common/event-dispatcher/owox-e
 const connector_run_event_1 = require("../../events/connector-run.event");
 const project_billing_service_1 = require("../project-billing/project-billing.service");
 const connector_definition_service_1 = require("./connector-definition.service");
+const connector_run_trigger_service_1 = require("./connector-run-trigger.service");
+const backfill_segment_1 = require("./backfill-segment");
 const connector_process_spawner_service_1 = require("./connector-process-spawner.service");
 const connector_storage_config_service_1 = require("./connector-storage-config.service");
 const connector_source_config_service_1 = require("./connector-source-config.service");
@@ -69,8 +72,9 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
     connectorDefinitionService;
     connectorSourceCredentialsService;
     configService;
+    connectorRunTriggerService;
     logger = new common_1.Logger(ConnectorExecutorService_1.name);
-    constructor(dataMartRunRepository, processSpawner, storageConfigService, sourceConfigService, credentialInjector, connectorOutputCaptureService, connectorStateService, gracefulShutdownService, systemTimeService, eventDispatcher, projectBillingService, dataMartService, connectorDefinitionService, connectorSourceCredentialsService, configService) {
+    constructor(dataMartRunRepository, processSpawner, storageConfigService, sourceConfigService, credentialInjector, connectorOutputCaptureService, connectorStateService, gracefulShutdownService, systemTimeService, eventDispatcher, projectBillingService, dataMartService, connectorDefinitionService, connectorSourceCredentialsService, configService, connectorRunTriggerService) {
         this.dataMartRunRepository = dataMartRunRepository;
         this.processSpawner = processSpawner;
         this.storageConfigService = storageConfigService;
@@ -86,6 +90,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
         this.connectorDefinitionService = connectorDefinitionService;
         this.connectorSourceCredentialsService = connectorSourceCredentialsService;
         this.configService = configService;
+        this.connectorRunTriggerService = connectorRunTriggerService;
     }
     async executeInBackground(dataMart, run, payload, signal) {
         const runId = run.id;
@@ -102,6 +107,8 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
         let hasAnySuccessfulConfiguration = false;
         let wasCancelled = false;
         let operationBlockedException;
+        const chainMeta = (0, manual_backfill_range_1.readBackfillChainMeta)(run.additionalParams);
+        const segmentWindow = chainMeta ? (0, manual_backfill_range_1.readBackfillRange)((0, manual_backfill_range_1.unwrapRunPayload)(payload)) : undefined;
         try {
             if (this.gracefulShutdownService.isInShutdownMode()) {
                 throw new connector_execution_error_1.ConnectorExecutionError('Skipping connector execution. Application is shutting down.', undefined, { dataMartId: dataMart.id, projectId: dataMart.projectId, runId });
@@ -119,7 +126,12 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
             logFlusher = this.createRunLogFlusher(runId, liveLogs, liveErrors, hasOutputFromEarlierAttempt);
             logFlusher?.start();
-            configurationResults = await this.runConnectorConfigurations(run, processId, dataMart, payload, signal, liveLogs, liveErrors);
+            if (chainMeta && segmentWindow) {
+                const startedMarker = (0, backfill_segment_1.buildSegmentMarker)(chainMeta, segmentWindow, 'started', this.systemTimeService.now().toISOString());
+                addBoundedMessage(capturedLogs, startedMarker);
+                addBoundedMessage(liveLogs, startedMarker);
+            }
+            configurationResults = await this.runConnectorConfigurations(run, processId, dataMart, payload, signal, liveLogs, liveErrors, chainMeta !== undefined);
             configurationResults.forEach(result => {
                 result.logs.forEach(log => addBoundedMessage(capturedLogs, log));
                 result.errors.forEach(error => addBoundedMessage(capturedErrors, error));
@@ -155,6 +167,37 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
         }
         finally {
+            const isShuttingDown = this.gracefulShutdownService.isInShutdownMode();
+            const willBeInterrupted = isShuttingDown && !wasCancelled && !allConfigurationsSucceeded;
+            const anyDateProgress = configurationResults.some(result => result.backfillLoadedThrough !== undefined);
+            const chainStep = chainMeta &&
+                !wasCancelled &&
+                signal?.aborted !== true &&
+                !operationBlockedException &&
+                !willBeInterrupted
+                ? (0, manual_backfill_range_1.nextBackfillChainStep)(chainMeta, {
+                    segmentSucceeded: allConfigurationsSucceeded,
+                    anyProgressAdvanced: anyDateProgress,
+                }, (0, manual_backfill_range_1.getMaxManualBackfillDays)())
+                : undefined;
+            const chainContinues = chainStep?.action === 'continue';
+            if (chainMeta && segmentWindow && !wasCancelled && !willBeInterrupted) {
+                const reportedDays = configurationResults.flatMap(result => result.backfillLoadedThrough ? [result.backfillLoadedThrough] : []);
+                const loadedThrough = reportedDays.length > 0 ? reportedDays.reduce((a, b) => (a < b ? a : b)) : undefined;
+                const chainEnd = operationBlockedException
+                    ? 'stopped'
+                    : chainStep?.action === 'end'
+                        ? chainStep.chainEnd
+                        : undefined;
+                (0, connector_message_utils_1.addMessageToArray)(capturedLogs, (0, backfill_segment_1.buildSegmentMarker)(chainMeta, segmentWindow, allConfigurationsSucceeded ? 'completed' : 'failed', this.systemTimeService.now().toISOString(), {
+                    loadedThrough,
+                    configurationsLoaded: reportedDays.length,
+                    configurationsTotal: configurationResults.length,
+                    counted: hasAnySuccessfulConfiguration,
+                    chainContinues,
+                    ...(chainEnd ? { chainEnd } : {}),
+                }));
+            }
             const hasSuccessfulFieldsUpdate = configurationResults.some(result => result.success && result.fieldsUpdate);
             try {
                 await this.persistSuccessfulFieldsUpdate(dataMart, configurationResults, runId);
@@ -180,21 +223,21 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
             await logFlusher?.stop();
             const flushedSnapshot = logFlusher?.persistedSnapshot() ?? null;
-            const persistedStatus = await this.updateRunStatus(runId, allConfigurationsSucceeded, capturedLogs, capturedErrors, operationBlockedException, wasCancelled, flushedSnapshot);
+            const persistedStatus = await this.updateRunStatus(runId, allConfigurationsSucceeded, capturedLogs, capturedErrors, operationBlockedException, wasCancelled, willBeInterrupted, flushedSnapshot, chainStep?.action === 'continue'
+                ? { run, dataMart, meta: chainStep.meta, window: chainStep.window }
+                : undefined);
             if (hasAnySuccessfulConfiguration &&
                 persistedStatus !== null &&
                 persistedStatus !== data_mart_run_status_enum_1.DataMartRunStatus.INTERRUPTED) {
                 await this.projectBillingService.registerConnectorRunConsumption(dataMart, runId);
             }
-            if (allConfigurationsSucceeded && persistedStatus !== null) {
+            if (persistedStatus === data_mart_run_status_enum_1.DataMartRunStatus.SUCCESS) {
                 await this.eventDispatcher.publishExternal(new connector_run_event_1.ConnectorRunEvent(dataMart.id, runId, dataMart.projectId, run.createdById ?? 'system', run.runType, 'successfully'));
             }
-            else if (persistedStatus !== null &&
-                !wasCancelled &&
-                !this.gracefulShutdownService.isInShutdownMode()) {
+            else if (persistedStatus !== null && !wasCancelled && !willBeInterrupted) {
                 await this.eventDispatcher.publishExternal(new connector_run_event_1.ConnectorRunEvent(dataMart.id, runId, dataMart.projectId, run.createdById ?? 'system', run.runType, 'unsuccessfully'));
             }
-            if (!hasSuccessfulFieldsUpdate) {
+            if (!hasSuccessfulFieldsUpdate && !chainContinues) {
                 await this.actualizeSchemaAfterConnectorExecution(dataMart, runId);
             }
             this.gracefulShutdownService.unregisterActiveProcess(processId);
@@ -225,7 +268,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             }
         }
     }
-    async runConnectorConfigurations(run, processId, dataMart, payload, signal, liveLogs = [], liveErrors = []) {
+    async runConnectorConfigurations(run, processId, dataMart, payload, signal, liveLogs = [], liveErrors = [], isChainSegment = false) {
         const runId = run.id;
         const definition = dataMart.definition;
         const { connector } = definition;
@@ -549,6 +592,9 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                     logs: configLogs,
                     errors: configErrors,
                     fieldsUpdate,
+                    ...(isChainSegment && backfillProgress[configId] !== undefined
+                        ? { backfillLoadedThrough: backfillProgress[configId] }
+                        : {}),
                 });
             }
         }
@@ -742,7 +788,7 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
             });
         }, error => this.logger.warn(`Incremental log flush failed for run ${runId}: ${(0, internal_helpers_1.castError)(error).message}`));
     }
-    async updateRunStatus(runId, allConfigurationsSucceeded, capturedLogs, capturedErrors, operationBlockedException, wasCancelled = false, flushedSnapshot = null) {
+    async updateRunStatus(runId, allConfigurationsSucceeded, capturedLogs, capturedErrors, operationBlockedException, wasCancelled = false, willBeInterrupted = false, flushedSnapshot = null, nextSegment) {
         let status = wasCancelled
             ? data_mart_run_status_enum_1.DataMartRunStatus.CANCELLED
             : allConfigurationsSucceeded
@@ -750,21 +796,37 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
                 : operationBlockedException
                     ? data_mart_run_status_enum_1.DataMartRunStatus.RESTRICTED
                     : data_mart_run_status_enum_1.DataMartRunStatus.FAILED;
-        if (!wasCancelled &&
-            !allConfigurationsSucceeded &&
-            this.gracefulShutdownService.isInShutdownMode()) {
+        if (willBeInterrupted) {
             status = data_mart_run_status_enum_1.DataMartRunStatus.INTERRUPTED;
         }
         const newLogStrings = capturedLogs.map(log => JSON.stringify(log));
         const newErrorStrings = capturedErrors.map(error => JSON.stringify(error));
-        const { logs: logsToSave, errors: errorsToSave } = await this.mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings, flushedSnapshot);
-        const result = await this.dataMartRunRepository.update({ id: runId, status: (0, typeorm_2.In)(data_mart_run_cancellation_1.NON_TERMINAL_DATA_MART_RUN_STATUSES) }, {
-            status,
-            finishedAt: this.systemTimeService.now(),
-            logs: logsToSave,
-            errors: errorsToSave,
-        });
-        if (result.affected) {
+        const merged = await this.mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings, flushedSnapshot);
+        let logsToSave = merged.logs;
+        const errorsToSave = merged.errors;
+        let landed;
+        try {
+            landed = await this.writeRunOutcome(runId, status, logsToSave, errorsToSave, nextSegment);
+        }
+        catch (error) {
+            if (!nextSegment)
+                throw error;
+            const failure = (0, internal_helpers_1.castError)(error);
+            this.logger.error(`Failed to queue the next backfill segment: ${failure.message}`, failure.stack, {
+                runId,
+            });
+            const queueWarning = 'Could not queue the next backfill segment. The backfill continues automatically within 15 minutes.';
+            logsToSave = this.capMergedEntries([
+                ...(logsToSave ?? []),
+                JSON.stringify({
+                    type: connector_message_type_enum_1.ConnectorMessageType.WARNING,
+                    at: this.systemTimeService.now().toISOString(),
+                    warning: queueWarning,
+                }),
+            ]);
+            landed = await this.writeRunOutcome(runId, status, logsToSave, errorsToSave);
+        }
+        if (landed) {
             return status;
         }
         this.logger.log(`Skipped final status update for run ${runId}: run already reached a terminal status`);
@@ -773,6 +835,21 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
         }
         return null;
     }
+    async writeRunOutcome(runId, status, logs, errors, nextSegment) {
+        const result = await this.dataMartRunRepository.update({ id: runId, status: (0, typeorm_2.In)(data_mart_run_cancellation_1.NON_TERMINAL_DATA_MART_RUN_STATUSES) }, { status, finishedAt: this.systemTimeService.now(), logs, errors });
+        if (!result.affected)
+            return false;
+        if (!nextSegment)
+            return true;
+        const { run, dataMart, meta, window } = nextSegment;
+        if (status !== data_mart_run_status_enum_1.DataMartRunStatus.SUCCESS && status !== data_mart_run_status_enum_1.DataMartRunStatus.FAILED) {
+            this.logger.warn(`Not queueing backfill segment ${meta.segmentIndex} of ${meta.segmentsTotal}: run ended ${status}`, { dataMartId: run.dataMartId, projectId: dataMart.projectId, runId });
+            return true;
+        }
+        const nextRun = await (0, backfill_segment_1.createNextSegmentRun)(this.dataMartRunRepository, this.connectorRunTriggerService, run, dataMart, meta, window);
+        this.logger.log(`Queued backfill segment ${meta.segmentIndex} of ${meta.segmentsTotal} as run ${nextRun.id}`, { dataMartId: run.dataMartId, projectId: dataMart.projectId, runId, nextRunId: nextRun.id });
+        return true;
+    }
     async mergeWithPersistedOutput(runId, newLogStrings, newErrorStrings, flushedSnapshot = null) {
         const existing = await this.dataMartRunRepository.findOne({ where: { id: runId } });
         const existingLogs = this.discountFlushedEntries(existing?.logs ?? [], flushedSnapshot?.logs, newLogStrings);
@@ -824,6 +901,12 @@ let ConnectorExecutorService = ConnectorExecutorService_1 = class ConnectorExecu
     }
 };
 exports.ConnectorExecutorService = ConnectorExecutorService;
+__decorate([
+    (0, typeorm_transactional_1.Transactional)(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, String, Object, Object, Object]),
+    __metadata("design:returntype", Promise)
+], ConnectorExecutorService.prototype, "writeRunOutcome", null);
 exports.ConnectorExecutorService = ConnectorExecutorService = ConnectorExecutorService_1 = __decorate([
     (0, common_1.Injectable)(),
     __param(0, (0, typeorm_1.InjectRepository)(data_mart_run_entity_1.DataMartRun)),
@@ -841,6 +924,7 @@ exports.ConnectorExecutorService = ConnectorExecutorService = ConnectorExecutorS
         data_mart_service_1.DataMartService,
         connector_definition_service_1.ConnectorDefinitionService,
         connector_source_credentials_service_1.ConnectorSourceCredentialsService,
-        config_1.ConfigService])
+        config_1.ConfigService,
+        connector_run_trigger_service_1.ConnectorRunTriggerService])
 ], ConnectorExecutorService);
 //# sourceMappingURL=connector-executor.service.js.map

package/dist/src/data-marts/services/connector/connector-run.service.d.ts

@@ -17,6 +17,9 @@ export declare class ConnectorRunService {
     executeExistingRun(dataMart: DataMart, run: DataMartRun, payload?: Record<string, unknown> | null, signal?: AbortSignal): Promise<void>;
     getDataMartConnectorRunsByStatus(status: DataMartRunStatus): Promise<DataMartRun[]>;
     executeInterruptedRuns(): Promise<void>;
+    repairBackfillChains(): Promise<void>;
+    private hasLaterRun;
+    private queueChainSegment;
     private validateDataMartForConnector;
     private checkDataMartIsRunning;
     private createDataMartRun;