@oronts/vendure-data-hub-plugin 0.1.0

package/dist/src/jobs/handlers/pipeline-run.handler.js

@@ -0,0 +1,150 @@
+"use strict";
+/**
+ * Pipeline Run Job Handler
+ *
+ * Async pipeline run execution via Vendure's job queue system.
+ * Includes retry logic configuration and proper error categorization.
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DataHubRunQueueHandler = void 0;
+const common_1 = require("@nestjs/common");
+const core_1 = require("@vendure/core");
+const index_1 = require("../../constants/index");
+const pipeline_runner_service_1 = require("../../services/pipeline/pipeline-runner.service");
+const logger_1 = require("../../services/logger");
+const pipeline_events_1 = require("../../services/events/pipeline-events");
+const job_processor_1 = require("../processors/job-processor");
+const error_utils_1 = require("../../utils/error.utils");
+/**
+ * Default job queue configuration for pipeline runs
+ */
+const PIPELINE_JOB_DEFAULTS = {
+    /** Default number of retries for failed jobs */
+    RETRIES: index_1.HTTP.MAX_RETRIES,
+    /** Default backoff delay in milliseconds */
+    BACKOFF_DELAY_MS: index_1.HTTP.RETRY_DELAY_MS,
+};
+/**
+ * DataHub Run Queue Handler
+ *
+ * Creates and manages the job queue for pipeline run execution.
+ * Jobs are added when a pipeline run is triggered and processed
+ * asynchronously by the PipelineRunnerService.
+ */
+let DataHubRunQueueHandler = class DataHubRunQueueHandler {
+    constructor(jobQueueService, eventBus, runner, loggerFactory) {
+        this.jobQueueService = jobQueueService;
+        this.eventBus = eventBus;
+        this.runner = runner;
+        this.logger = loggerFactory.createLogger(index_1.LOGGER_CONTEXTS.RUN_QUEUE_HANDLER);
+    }
+    /**
+     * Initialize the job queue on module startup
+     */
+    async onModuleInit() {
+        this.queue = await this.jobQueueService.createQueue({
+            name: index_1.QUEUE_NAMES.RUN,
+            process: async (job) => {
+                const startTime = Date.now();
+                const { runId } = job.data;
+                this.logger.debug('Processing pipeline run job', {
+                    runId,
+                    jobId: job.id,
+                });
+                try {
+                    await this.runner.execute(runId);
+                    const durationMs = Date.now() - startTime;
+                    this.logger.debug('Pipeline run job completed', {
+                        runId: String(runId),
+                        jobId: String(job.id),
+                        durationMs,
+                    });
+                }
+                catch (error) {
+                    const durationMs = Date.now() - startTime;
+                    const err = (0, error_utils_1.ensureError)(error);
+                    const isRetryable = (0, job_processor_1.isRetryableError)(err);
+                    this.logger.error('Pipeline run job failed', err, {
+                        runId,
+                        jobId: job.id,
+                        durationMs,
+                        isRetryable,
+                        attempt: job.attempts,
+                    });
+                    // Re-throw to let job queue handle retries
+                    // Non-retryable errors should still be thrown to mark job as failed
+                    throw error;
+                }
+            },
+        });
+        this.logger.info('Pipeline run job queue initialized', {
+            queueName: index_1.QUEUE_NAMES.RUN,
+        });
+        // Subscribe to PipelineQueueRequestEvent to handle queue requests
+        // This breaks the circular dependency: PipelineService -> EventBus -> DataHubRunQueueHandler
+        this.eventSubscription = this.eventBus.ofType(pipeline_events_1.PipelineQueueRequestEvent).subscribe(event => {
+            this.logger.debug('Received PipelineQueueRequestEvent', {
+                runId: event.runId,
+                pipelineId: event.pipelineId,
+                triggeredBy: event.triggeredBy,
+            });
+            this.enqueueRun(event.runId).catch(error => {
+                this.logger.error('Failed to enqueue run from event', (0, error_utils_1.ensureError)(error), { runId: event.runId, pipelineId: event.pipelineId });
+            });
+        });
+    }
+    /**
+     * Cleanup event subscription on module destroy
+     */
+    onModuleDestroy() {
+        if (this.eventSubscription) {
+            this.eventSubscription.unsubscribe();
+        }
+    }
+    /**
+     * Enqueue a pipeline run for async execution
+     *
+     * @param runId - The ID of the pipeline run to execute
+     * @param options - Optional job options (retries, priority)
+     */
+    async enqueueRun(runId, options) {
+        if (!runId) {
+            throw new Error('runId is required to enqueue a pipeline run');
+        }
+        this.logger.debug('Enqueueing pipeline run', {
+            runId,
+            retries: options?.retries ?? PIPELINE_JOB_DEFAULTS.RETRIES,
+        });
+        await this.queue.add({ runId }, {
+            retries: options?.retries ?? PIPELINE_JOB_DEFAULTS.RETRIES,
+        });
+    }
+    /**
+     * Get the underlying job queue (for advanced operations)
+     */
+    getQueue() {
+        return this.queue;
+    }
+};
+exports.DataHubRunQueueHandler = DataHubRunQueueHandler;
+exports.DataHubRunQueueHandler = DataHubRunQueueHandler = __decorate([
+    (0, common_1.Injectable)(),
+    __param(2, (0, common_1.Inject)((0, common_1.forwardRef)(() => pipeline_runner_service_1.PipelineRunnerService))),
+    __metadata("design:paramtypes", [core_1.JobQueueService,
+        core_1.EventBus,
+        pipeline_runner_service_1.PipelineRunnerService,
+        logger_1.DataHubLoggerFactory])
+], DataHubRunQueueHandler);
+//# sourceMappingURL=pipeline-run.handler.js.map

package/dist/src/jobs/handlers/pipeline-run.handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-run.handler.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/pipeline-run.handler.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;AAEH,2CAA+F;AAC/F,wCAAwE;AAExE,iDAA2E;AAC3E,6FAAwF;AACxF,kDAA4E;AAE5E,2EAAkF;AAClF,+DAA+D;AAC/D,yDAAsD;AAEtD;;GAEG;AACH,MAAM,qBAAqB,GAAG;IAC1B,gDAAgD;IAChD,OAAO,EAAE,YAAI,CAAC,WAAW;IACzB,4CAA4C;IAC5C,gBAAgB,EAAE,YAAI,CAAC,cAAc;CAC/B,CAAC;AAEX;;;;;;GAMG;AAEI,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAK/B,YACY,eAAgC,EAChC,QAAkB,EAElB,MAA6B,EACrC,aAAmC;QAJ3B,oBAAe,GAAf,eAAe,CAAiB;QAChC,aAAQ,GAAR,QAAQ,CAAU;QAElB,WAAM,GAAN,MAAM,CAAuB;QAGrC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,YAAY,CAAC,uBAAe,CAAC,iBAAiB,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CAAqB;YACpE,IAAI,EAAE,mBAAW,CAAC,GAAG;YACrB,OAAO,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;gBACjB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;gBAE3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE;oBAC7C,KAAK;oBACL,KAAK,EAAE,GAAG,CAAC,EAAE;iBAChB,CAAC,CAAC;gBAEH,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAEjC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;wBAC5C,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;wBACpB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;wBACrB,UAAU;qBACb,CAAC,CAAC;gBACP,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAC1C,MAAM,GAAG,GAAG,IAAA,yBAAW,EAAC,KAAK,CAAC,CAAC;oBAC/B,MAAM,WAAW,GAAG,IAAA,gCAAgB,EAAC,GAAG,CAAC,CAAC;oBAE1C,IAAI,CAAC,MAAM,CAAC,KAAK,CACb,yBAAyB,EACzB,GAAG,EACH;wBACI,KAAK;wBACL,KAAK,EAAE,GAAG,CAAC,EAAE;wBACb,UAAU;wBACV,WAAW;wBACX,OAAO,EAAE,GAAG,CAAC,QAAQ;qBACxB,CACJ,CAAC;oBAEF,2CAA2C;oBAC3C,oEAAoE;oBACpE,MAAM,KAAK,CAAC;gBAChB,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;YACnD,SAAS,EAAE,mBAAW,CAAC,GAAG;SAC7B,CAAC,CAAC;QAEH,kEAAkE;QAClE,6FAA6F;QAC7F,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,2CAAyB,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE;YACvF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE;gBACpD,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;aACjC,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CACb,kCAAkC,EAClC,IAAA,yBAAW,EAAC,KAAK,CAAC,EAClB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CACvD,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,eAAe;QACX,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAS,EAAE,OAAoB;QAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE;YACzC,KAAK;YACL,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,qBAAqB,CAAC,OAAO;SAC7D,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAChB,EAAE,KAAK,EAAE,EACT;YACI,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,qBAAqB,CAAC,OAAO;SAC7D,CACJ,CAAC;IACN,CAAC;IAED;;OAEG;IACH,QAAQ;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;CACJ,CAAA;AA5HY,wDAAsB;iCAAtB,sBAAsB;IADlC,IAAA,mBAAU,GAAE;IASJ,WAAA,IAAA,eAAM,EAAC,IAAA,mBAAU,EAAC,GAAG,EAAE,CAAC,+CAAqB,CAAC,CAAC,CAAA;qCAFvB,sBAAe;QACtB,eAAQ;QAEV,+CAAqB;QACtB,6BAAoB;GAV9B,sBAAsB,CA4HlC"}

package/dist/src/jobs/handlers/schedule.handler.d.ts

@@ -0,0 +1,71 @@
+import { OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import { RequestContextService, TransactionalConnection } from '@vendure/core';
+import { PipelineService } from '../../services/pipeline/pipeline.service';
+import { RuntimeConfigService } from '../../services/runtime/runtime-config.service';
+import { DistributedLockService } from '../../services/runtime/distributed-lock.service';
+import { DataHubLoggerFactory } from '../../services/logger';
+import { DomainEventsService } from '../../services/events/domain-events.service';
+/**
+ * DataHub Schedule Handler
+ *
+ * Manages scheduled pipeline execution by:
+ * - Polling for enabled pipelines with schedule triggers
+ * - Setting up interval-based timers
+ * - Setting up cron-based timers with minute-level precision
+ * - Automatically refreshing schedules when pipelines change
+ * - Preventing concurrent runs of the same pipeline (concurrency control)
+ */
+export declare class DataHubScheduleHandler implements OnModuleInit, OnModuleDestroy {
+    private connection;
+    private requestContextService;
+    private pipelineService;
+    private runtimeConfigService;
+    private domainEvents;
+    private distributedLock?;
+    private readonly logger;
+    private readonly schedulerConfig;
+    private timers;
+    private lastCronKeyByPipeline;
+    /** Mutex flag to prevent concurrent refresh operations */
+    private isRefreshing;
+    /** Flag to track if module is being destroyed */
+    private isDestroying;
+    /** Track consecutive failures per pipeline for circuit breaker pattern */
+    private failureCountByPipeline;
+    /** Maximum consecutive failures before pausing a pipeline's schedule */
+    private readonly maxConsecutiveFailures;
+    /** Safety cap for tracking maps to prevent unbounded memory growth */
+    private readonly maxTrackingEntries;
+    constructor(connection: TransactionalConnection, requestContextService: RequestContextService, pipelineService: PipelineService, runtimeConfigService: RuntimeConfigService, domainEvents: DomainEventsService, loggerFactory: DataHubLoggerFactory, distributedLock?: DistributedLockService | undefined);
+    onModuleInit(): Promise<void>;
+    onModuleDestroy(): Promise<void>;
+    /** Uses a mutex to prevent concurrent refresh operations */
+    private refresh;
+    private cleanupStaleCronKeys;
+    private cleanupStaleFailureCounts;
+    private isPipelineRunning;
+    private setupIntervalSchedule;
+    private setupCronSchedule;
+    /** Uses distributed locks to prevent simultaneous runs across instances */
+    private triggerPipeline;
+    forceRefresh(): Promise<void>;
+    getActiveScheduleCount(): number;
+    getScheduledPipelines(): string[];
+    clearCronKeyForPipeline(code: string): void;
+    getCronKeyCount(): number;
+    resetCircuitBreaker(code: string): void;
+    resetAllCircuitBreakers(): void;
+    getCircuitBreakerStatus(): Map<string, {
+        failureCount: number;
+        isPaused: boolean;
+    }>;
+    getHealthMetrics(): {
+        activeTimers: number;
+        cronKeyCount: number;
+        trackedFailures: number;
+        pausedPipelines: number;
+        isRefreshing: boolean;
+        isDestroying: boolean;
+    };
+}
+//# sourceMappingURL=schedule.handler.d.ts.map

package/dist/src/jobs/handlers/schedule.handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schedule.handler.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/schedule.handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,eAAe,EAAE,YAAY,EAAY,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAM,MAAM,eAAe,CAAC;AAGnF,OAAO,EAAE,eAAe,EAAE,MAAM,0CAA0C,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+CAA+C,CAAC;AACrF,OAAO,EAAE,sBAAsB,EAAE,MAAM,iDAAiD,CAAC;AACzF,OAAO,EAAiB,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAE,mBAAmB,EAAE,MAAM,6CAA6C,CAAC;AAiClF;;;;;;;;;GASG;AACH,qBACa,sBAAuB,YAAW,YAAY,EAAE,eAAe;IAiBpE,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,qBAAqB;IAC7B,OAAO,CAAC,eAAe;IACvB,OAAO,CAAC,oBAAoB;IAC5B,OAAO,CAAC,YAAY;IAER,OAAO,CAAC,eAAe,CAAC;IAtBxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA4B;IAC5D,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,qBAAqB,CAA6B;IAC1D,0DAA0D;IAC1D,OAAO,CAAC,YAAY,CAAS;IAC7B,iDAAiD;IACjD,OAAO,CAAC,YAAY,CAAS;IAC7B,0EAA0E;IAC1E,OAAO,CAAC,sBAAsB,CAA6B;IAC3D,wEAAwE;IACxE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAK;IAC5C,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAQ;gBAG/B,UAAU,EAAE,uBAAuB,EACnC,qBAAqB,EAAE,qBAAqB,EAC5C,eAAe,EAAE,eAAe,EAChC,oBAAoB,EAAE,oBAAoB,EAC1C,YAAY,EAAE,mBAAmB,EACzC,aAAa,EAAE,oBAAoB,EACf,eAAe,CAAC,EAAE,sBAAsB,YAAA;IAM1D,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B7B,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBtC,4DAA4D;YAC9C,OAAO;IAoHrB,OAAO,CAAC,oBAAoB;IAyB5B,OAAO,CAAC,yBAAyB;YAyBnB,iBAAiB;IAc/B,OAAO,CAAC,qBAAqB;IAmC7B,OAAO,CAAC,iBAAiB;IA0EzB,2EAA2E;YAC7D,eAAe;IAiHvB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAInC,sBAAsB,IAAI,MAAM;IAIhC,qBAAqB,IAAI,MAAM,EAAE;IAMjC,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAO3C,eAAe,IAAI,MAAM;IAIzB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAYvC,uBAAuB,IAAI,IAAI;IAQ/B,uBAAuB,IAAI,GAAG,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC;IAWnF,gBAAgB,IAAI;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,OAAO,CAAC;QACtB,YAAY,EAAE,OAAO,CAAC;KACzB;CAiBJ"}