@oronts/vendure-data-hub-plugin 0.1.0

package/dist/src/services/logger/execution-logger.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Centralized database logging with configurable persistence levels.
+ * Respects LogPersistenceLevel to control what gets persisted while
+ * always maintaining full console logging.
+ */
+import { RequestContext, ID } from '@vendure/core';
+import { LogPersistenceLevel } from '../../constants/enums';
+import { PipelineLogService } from '../pipeline/pipeline-log.service';
+import { DataHubSettingsService } from '../config/settings.service';
+import { DataHubLoggerFactory } from './datahub-logger';
+import type { JsonObject, JsonValue } from '../../types/index';
+/** Log event types for categorization */
+export type LogEventType = 'pipeline.start' | 'pipeline.complete' | 'pipeline.fail' | 'step.start' | 'step.complete' | 'step.fail' | 'record.error' | 'transform.mapping' | 'extract.source' | 'load.target' | 'debug';
+/** Options for logging an event */
+interface LogEventOptions {
+    pipelineId?: ID;
+    runId?: ID;
+    stepKey?: string;
+    durationMs?: number;
+    recordsProcessed?: number;
+    recordsFailed?: number;
+    recordsIn?: number;
+    recordsOut?: number;
+    context?: JsonObject;
+    metadata?: JsonObject;
+}
+/** Source/target mapping info for debugging */
+interface FieldMappingInfo {
+    sourceField: string;
+    targetField: string;
+    transformType?: string;
+    sampleSourceValue?: JsonValue;
+    sampleTargetValue?: JsonValue;
+}
+/** Step execution info for detailed logging */
+interface StepExecutionInfo {
+    stepKey: string;
+    stepType: string;
+    adapterCode?: string;
+    recordsIn: number;
+    recordsOut: number;
+    succeeded: number;
+    failed: number;
+    durationMs: number;
+    sampleRecord?: JsonObject;
+    fieldMappings?: FieldMappingInfo[];
+}
+export declare class ExecutionLogger {
+    private pipelineLogService;
+    private settingsService;
+    private readonly consoleLogger;
+    private cachedLevel;
+    private cacheTime;
+    private readonly cacheTtlMs;
+    constructor(pipelineLogService: PipelineLogService, settingsService: DataHubSettingsService, loggerFactory: DataHubLoggerFactory);
+    /**
+     * Get the current persistence level (with caching)
+     */
+    private getPersistenceLevel;
+    /**
+     * Check if an event should be persisted to database
+     */
+    private shouldPersist;
+    /**
+     * Log a pipeline start event
+     */
+    logPipelineStart(ctx: RequestContext, pipelineCode: string, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a pipeline completion event
+     */
+    logPipelineComplete(ctx: RequestContext, pipelineCode: string, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a pipeline failure event
+     */
+    logPipelineFailed(ctx: RequestContext, pipelineCode: string, error: Error, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a step start event
+     */
+    logStepStart(ctx: RequestContext, stepKey: string, stepType: string, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a step completion event
+     */
+    logStepComplete(ctx: RequestContext, stepKey: string, stepType: string, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a step failure event
+     */
+    logStepFailed(ctx: RequestContext, stepKey: string, stepType: string, error: Error, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a record-level error
+     */
+    logRecordError(ctx: RequestContext, stepKey: string, errorMessage: string, payload: Record<string, unknown>, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a debug event (only persisted at DEBUG level)
+     */
+    logDebug(ctx: RequestContext, message: string, options: LogEventOptions): Promise<void>;
+    /**
+     * Log detailed step execution info (STEP level)
+     */
+    logStepExecution(ctx: RequestContext, info: StepExecutionInfo, options: LogEventOptions): Promise<void>;
+    /**
+     * Log extracted source data (DEBUG level)
+     */
+    logExtractedData(ctx: RequestContext, stepKey: string, adapterCode: string, records: Record<string, unknown>[], options: LogEventOptions): Promise<void>;
+    /**
+     * Log data before loading to target (DEBUG level)
+     */
+    logLoadTargetData(ctx: RequestContext, stepKey: string, adapterCode: string, records: Record<string, unknown>[], options: LogEventOptions): Promise<void>;
+    /**
+     * Log field mappings from transform step (DEBUG level)
+     * Sample values are sanitized to remove sensitive data.
+     */
+    logFieldMappings(ctx: RequestContext, stepKey: string, adapterCode: string, inputRecord: Record<string, unknown>, outputRecord: Record<string, unknown>, options: LogEventOptions): Promise<void>;
+    /**
+     * Log a comparison between source and target for a record (DEBUG level)
+     * Records are automatically sanitized to remove sensitive data.
+     */
+    logRecordTransformation(ctx: RequestContext, stepKey: string, recordIndex: number, sourceRecord: Record<string, unknown>, targetRecord: Record<string, unknown>, options: LogEventOptions): Promise<void>;
+    /**
+     * Truncate and sanitize a sample record for safe logging.
+     * Removes sensitive fields and masks PII (emails, phone numbers).
+     */
+    private truncateSample;
+    /**
+     * Truncate a value for safe logging
+     */
+    private truncateValue;
+    /**
+     * Invalidate the cached persistence level (call when settings change)
+     */
+    invalidateCache(): void;
+    /**
+     * Get current persistence level (for external checks)
+     */
+    getCurrentLevel(): Promise<LogPersistenceLevel>;
+}
+export {};
+//# sourceMappingURL=execution-logger.d.ts.map

package/dist/src/services/logger/execution-logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-logger.d.ts","sourceRoot":"","sources":["../../../../src/services/logger/execution-logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EAAiB,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAEvE,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE/D,yCAAyC;AACzC,MAAM,MAAM,YAAY,GAClB,gBAAgB,GAChB,mBAAmB,GACnB,eAAe,GACf,YAAY,GACZ,eAAe,GACf,WAAW,GACX,cAAc,GACd,mBAAmB,GACnB,gBAAgB,GAChB,aAAa,GACb,OAAO,CAAC;AAEd,mCAAmC;AACnC,UAAU,eAAe;IACrB,UAAU,CAAC,EAAE,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,EAAE,CAAC;IACX,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,+CAA+C;AAC/C,UAAU,gBAAgB;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,SAAS,CAAC;IAC9B,iBAAiB,CAAC,EAAE,SAAS,CAAC;CACjC;AAED,+CAA+C;AAC/C,UAAU,iBAAiB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,aAAa,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACtC;AAiCD,qBACa,eAAe;IAOpB,OAAO,CAAC,kBAAkB;IAC1B,OAAO,CAAC,eAAe;IAP3B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,WAAW,CAAoC;IACvD,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;gBAGxC,kBAAkB,EAAE,kBAAkB,EACtC,eAAe,EAAE,sBAAsB,EAC/C,aAAa,EAAE,oBAAoB;IAKvC;;OAEG;YACW,mBAAmB;IAUjC;;OAEG;IACH,OAAO,CAAC,aAAa;IAKrB;;OAEG;IACG,gBAAgB,CAClB,GAAG,EAAE,cAAc,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAiBhB;;OAEG;IACG,mBAAmB,CACrB,GAAG,EAAE,cAAc,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IA2BhB;;OAEG;IACG,iBAAiB,CACnB,GAAG,EAAE,cAAc,EACnB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAuBhB;;OAEG;IACG,YAAY,CACd,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAkBhB;;OAEG;IACG,eAAe,CACjB,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IA6BhB;;OAEG;IACG,aAAa,CACf,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAyBhB;;OAEG;IACG,cAAc,CAChB,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAuBhB;;OAEG;IACG,QAAQ,CACV,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAiBhB;;OAEG;IACG,gBAAgB,CAClB,GAAG,EAAE,cAAc,EACnB,IAAI,EAAE,iBAAiB,EACvB,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IA2ChB;;OAEG;IACG,gBAAgB,CAClB,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAClC,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAmChB;;OAEG;IACG,iBAAiB,CACnB,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAClC,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IAmChB;;;OAGG;IACG,gBAAgB,CAClB,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IA8DhB;;;OAGG;IACG,uBAAuB,CACzB,GAAG,EAAE,cAAc,EACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,OAAO,EAAE,eAAe,GACzB,OAAO,CAAC,IAAI,CAAC;IA2BhB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAWtB;;OAEG;IACH,OAAO,CAAC,aAAa;IA0BrB;;OAEG;IACH,eAAe,IAAI,IAAI;IAKvB;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,mBAAmB,CAAC;CAGxD"}

package/dist/src/services/logger/execution-logger.js

@@ -0,0 +1,525 @@
+"use strict";
+/**
+ * Centralized database logging with configurable persistence levels.
+ * Respects LogPersistenceLevel to control what gets persisted while
+ * always maintaining full console logging.
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExecutionLogger = void 0;
+const common_1 = require("@nestjs/common");
+const enums_1 = require("../../constants/enums");
+const index_1 = require("../../constants/index");
+const pipeline_log_service_1 = require("../pipeline/pipeline-log.service");
+const settings_service_1 = require("../config/settings.service");
+const datahub_logger_1 = require("./datahub-logger");
+const sanitizer_1 = require("./sanitizer");
+/**
+ * Determines which persistence level is required for each event type
+ */
+const EVENT_LEVEL_MAP = {
+    'pipeline.start': enums_1.LogPersistenceLevel.PIPELINE,
+    'pipeline.complete': enums_1.LogPersistenceLevel.PIPELINE,
+    'pipeline.fail': enums_1.LogPersistenceLevel.ERROR_ONLY,
+    'step.start': enums_1.LogPersistenceLevel.STEP,
+    'step.complete': enums_1.LogPersistenceLevel.STEP,
+    'step.fail': enums_1.LogPersistenceLevel.ERROR_ONLY,
+    'record.error': enums_1.LogPersistenceLevel.ERROR_ONLY,
+    'transform.mapping': enums_1.LogPersistenceLevel.DEBUG,
+    'extract.source': enums_1.LogPersistenceLevel.DEBUG,
+    'load.target': enums_1.LogPersistenceLevel.DEBUG,
+    'debug': enums_1.LogPersistenceLevel.DEBUG,
+};
+/**
+ * Persistence level hierarchy (higher includes lower)
+ */
+const LEVEL_HIERARCHY = {
+    [enums_1.LogPersistenceLevel.ERROR_ONLY]: 1,
+    [enums_1.LogPersistenceLevel.PIPELINE]: 2,
+    [enums_1.LogPersistenceLevel.STEP]: 3,
+    [enums_1.LogPersistenceLevel.DEBUG]: 4,
+};
+const MAX_SAMPLE_SIZE = index_1.TRUNCATION.SAMPLE_VALUES_LIMIT;
+const MAX_MAPPINGS_LOG = 50;
+const MAX_FIELDS_LOG = 20;
+let ExecutionLogger = class ExecutionLogger {
+    constructor(pipelineLogService, settingsService, loggerFactory) {
+        this.pipelineLogService = pipelineLogService;
+        this.settingsService = settingsService;
+        this.cachedLevel = null;
+        this.cacheTime = 0;
+        this.cacheTtlMs = index_1.CACHE.SETTINGS_TTL_MS;
+        this.consoleLogger = loggerFactory.createLogger(index_1.LOGGER_CONTEXTS.EXECUTION_LOGGER);
+    }
+    /**
+     * Get the current persistence level (with caching)
+     */
+    async getPersistenceLevel() {
+        const now = Date.now();
+        if (this.cachedLevel && now - this.cacheTime < this.cacheTtlMs) {
+            return this.cachedLevel;
+        }
+        this.cachedLevel = await this.settingsService.getLogPersistenceLevel();
+        this.cacheTime = now;
+        return this.cachedLevel;
+    }
+    /**
+     * Check if an event should be persisted to database
+     */
+    shouldPersist(eventType, currentLevel) {
+        const requiredLevel = EVENT_LEVEL_MAP[eventType];
+        return LEVEL_HIERARCHY[currentLevel] >= LEVEL_HIERARCHY[requiredLevel];
+    }
+    /**
+     * Log a pipeline start event
+     */
+    async logPipelineStart(ctx, pipelineCode, options) {
+        const message = `Pipeline "${pipelineCode}" execution started`;
+        // Always log to console
+        this.consoleLogger.info(message, { pipelineCode, ...options.context });
+        // Persist to database based on level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('pipeline.start', level)) {
+            await this.pipelineLogService.info(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                context: { pipelineCode, ...options.context },
+            });
+        }
+    }
+    /**
+     * Log a pipeline completion event
+     */
+    async logPipelineComplete(ctx, pipelineCode, options) {
+        const message = `Pipeline "${pipelineCode}" execution completed`;
+        // Always log to console
+        this.consoleLogger.info(message, {
+            pipelineCode,
+            durationMs: options.durationMs,
+            recordsProcessed: options.recordsProcessed,
+            recordsFailed: options.recordsFailed,
+            ...options.context,
+        });
+        // Persist to database based on level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('pipeline.complete', level)) {
+            await this.pipelineLogService.info(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                durationMs: options.durationMs,
+                recordsProcessed: options.recordsProcessed,
+                recordsFailed: options.recordsFailed,
+                context: { pipelineCode, ...options.context },
+                metadata: options.metadata,
+            });
+        }
+    }
+    /**
+     * Log a pipeline failure event
+     */
+    async logPipelineFailed(ctx, pipelineCode, error, options) {
+        const message = `Pipeline "${pipelineCode}" execution failed: ${error.message}`;
+        // Always log to console
+        this.consoleLogger.error(message, error, {
+            pipelineCode,
+            durationMs: options.durationMs,
+            ...options.context,
+        });
+        // Always persist errors (ERROR_ONLY is minimum level)
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('pipeline.fail', level)) {
+            await this.pipelineLogService.error(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                durationMs: options.durationMs,
+                context: { pipelineCode, error: error.message, ...options.context },
+                metadata: { stack: error.stack ?? null, ...options.metadata },
+            });
+        }
+    }
+    /**
+     * Log a step start event
+     */
+    async logStepStart(ctx, stepKey, stepType, options) {
+        const message = `Step "${stepKey}" (${stepType}) started`;
+        // Always log to console
+        this.consoleLogger.debug(message, { stepKey, stepType, ...options.context });
+        // Persist to database based on level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('step.start', level)) {
+            await this.pipelineLogService.info(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                context: { stepType, ...options.context },
+            });
+        }
+    }
+    /**
+     * Log a step completion event
+     */
+    async logStepComplete(ctx, stepKey, stepType, options) {
+        const message = `Step "${stepKey}" (${stepType}) completed`;
+        // Always log to console
+        this.consoleLogger.debug(message, {
+            stepKey,
+            stepType,
+            durationMs: options.durationMs,
+            recordsProcessed: options.recordsProcessed,
+            recordsFailed: options.recordsFailed,
+            ...options.context,
+        });
+        // Persist to database based on level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('step.complete', level)) {
+            await this.pipelineLogService.info(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                durationMs: options.durationMs,
+                recordsProcessed: options.recordsProcessed,
+                recordsFailed: options.recordsFailed,
+                context: { stepType, ...options.context },
+                metadata: options.metadata,
+            });
+        }
+    }
+    /**
+     * Log a step failure event
+     */
+    async logStepFailed(ctx, stepKey, stepType, error, options) {
+        const message = `Step "${stepKey}" (${stepType}) failed: ${error.message}`;
+        // Always log to console
+        this.consoleLogger.error(message, error, {
+            stepKey,
+            stepType,
+            durationMs: options.durationMs,
+            ...options.context,
+        });
+        // Always persist errors
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('step.fail', level)) {
+            await this.pipelineLogService.error(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                durationMs: options.durationMs,
+                context: { stepType, error: error.message, ...options.context },
+                metadata: { stack: error.stack ?? null, ...options.metadata },
+            });
+        }
+    }
+    /**
+     * Log a record-level error
+     */
+    async logRecordError(ctx, stepKey, errorMessage, payload, options) {
+        const message = `Record error in step "${stepKey}": ${errorMessage}`;
+        // Always log to console
+        this.consoleLogger.warn(message, {
+            stepKey,
+            error: errorMessage,
+            ...options.context,
+        });
+        // Always persist record errors (with sanitized payload)
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('record.error', level)) {
+            await this.pipelineLogService.warn(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                context: { error: errorMessage, ...options.context },
+                metadata: (0, sanitizer_1.sanitizeRecord)(payload),
+            });
+        }
+    }
+    /**
+     * Log a debug event (only persisted at DEBUG level)
+     */
+    async logDebug(ctx, message, options) {
+        // Always log to console
+        this.consoleLogger.debug(message, options.context);
+        // Only persist at DEBUG level (with sanitized metadata)
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('debug', level)) {
+            await this.pipelineLogService.debug(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey: options.stepKey,
+                context: options.context ? (0, sanitizer_1.sanitizeForLog)(options.context) : undefined,
+                metadata: options.metadata ? (0, sanitizer_1.sanitizeForLog)(options.metadata) : undefined,
+            });
+        }
+    }
+    /**
+     * Log detailed step execution info (STEP level)
+     */
+    async logStepExecution(ctx, info, options) {
+        const throughput = (0, index_1.calculateThroughput)(info.recordsIn, info.durationMs);
+        const message = `Step "${info.stepKey}" (${info.stepType}) completed: ${info.recordsIn} in → ${info.recordsOut} out, ${info.succeeded} ok, ${info.failed} failed [${info.durationMs}ms, ${throughput} rec/s]`;
+        // Always log to console with full details
+        this.consoleLogger.info(message, {
+            stepKey: info.stepKey,
+            stepType: info.stepType,
+            adapterCode: info.adapterCode,
+            recordsIn: info.recordsIn,
+            recordsOut: info.recordsOut,
+            succeeded: info.succeeded,
+            failed: info.failed,
+            durationMs: info.durationMs,
+            throughput,
+        });
+        // Persist to database based on level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('step.complete', level)) {
+            await this.pipelineLogService.info(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey: info.stepKey,
+                durationMs: info.durationMs,
+                recordsProcessed: info.recordsIn,
+                recordsFailed: info.failed,
+                context: {
+                    stepType: info.stepType,
+                    adapterCode: info.adapterCode ?? null,
+                    recordsOut: info.recordsOut,
+                    succeeded: info.succeeded,
+                    throughput,
+                },
+                metadata: level === enums_1.LogPersistenceLevel.DEBUG ? {
+                    sampleRecord: info.sampleRecord ? this.truncateSample(info.sampleRecord) : null,
+                    fieldMappings: info.fieldMappings ?? null,
+                } : undefined,
+            });
+        }
+    }
+    /**
+     * Log extracted source data (DEBUG level)
+     */
+    async logExtractedData(ctx, stepKey, adapterCode, records, options) {
+        const sampleRecords = records.slice(0, MAX_SAMPLE_SIZE).map(r => this.truncateSample(r));
+        const fieldNames = records.length > 0 ? Object.keys(records[0]) : [];
+        const message = `Extract "${stepKey}" (${adapterCode}): ${records.length} records with ${fieldNames.length} fields`;
+        // Always log to console
+        this.consoleLogger.debug(message, {
+            stepKey,
+            adapterCode,
+            recordCount: records.length,
+            fieldCount: fieldNames.length,
+            fields: fieldNames.slice(0, MAX_FIELDS_LOG),
+        });
+        // Persist at DEBUG level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('extract.source', level)) {
+            await this.pipelineLogService.debug(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                context: {
+                    adapterCode,
+                    recordCount: records.length,
+                    fieldCount: fieldNames.length,
+                },
+                metadata: {
+                    fields: fieldNames,
+                    sampleRecords: sampleRecords,
+                },
+            });
+        }
+    }
+    /**
+     * Log data before loading to target (DEBUG level)
+     */
+    async logLoadTargetData(ctx, stepKey, adapterCode, records, options) {
+        const sampleRecords = records.slice(0, MAX_SAMPLE_SIZE).map(r => this.truncateSample(r));
+        const fieldNames = records.length > 0 ? Object.keys(records[0]) : [];
+        const message = `Load "${stepKey}" (${adapterCode}): ${records.length} records to load with ${fieldNames.length} fields`;
+        // Always log to console
+        this.consoleLogger.debug(message, {
+            stepKey,
+            adapterCode,
+            recordCount: records.length,
+            fieldCount: fieldNames.length,
+            fields: fieldNames.slice(0, MAX_FIELDS_LOG),
+        });
+        // Persist at DEBUG level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('load.target', level)) {
+            await this.pipelineLogService.debug(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                context: {
+                    adapterCode,
+                    recordCount: records.length,
+                    fieldCount: fieldNames.length,
+                },
+                metadata: {
+                    fields: fieldNames,
+                    sampleRecords: sampleRecords,
+                },
+            });
+        }
+    }
+    /**
+     * Log field mappings from transform step (DEBUG level)
+     * Sample values are sanitized to remove sensitive data.
+     */
+    async logFieldMappings(ctx, stepKey, adapterCode, inputRecord, outputRecord, options) {
+        // Sanitize input/output records before processing
+        const sanitizedInput = (0, sanitizer_1.sanitizeRecord)(inputRecord);
+        const sanitizedOutput = (0, sanitizer_1.sanitizeRecord)(outputRecord);
+        const inputFields = Object.keys(sanitizedInput);
+        const outputFields = Object.keys(sanitizedOutput);
+        // Build mapping info by comparing input and output
+        const mappings = [];
+        for (const outField of outputFields) {
+            const outValue = sanitizedOutput[outField];
+            // Try to find matching source field
+            let sourceField = outField;
+            if (!(outField in sanitizedInput)) {
+                // Check if value matches any input value
+                for (const inField of inputFields) {
+                    if (sanitizedInput[inField] === outValue) {
+                        sourceField = inField;
+                        break;
+                    }
+                }
+            }
+            mappings.push({
+                sourceField,
+                targetField: outField,
+                sampleSourceValue: this.truncateValue(sanitizedInput[sourceField]),
+                sampleTargetValue: this.truncateValue(outValue),
+            });
+        }
+        const message = `Transform "${stepKey}" (${adapterCode}): ${inputFields.length} input fields → ${outputFields.length} output fields`;
+        // Always log to console
+        this.consoleLogger.debug(message, {
+            stepKey,
+            adapterCode,
+            inputFieldCount: inputFields.length,
+            outputFieldCount: outputFields.length,
+        });
+        // Persist at DEBUG level
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('transform.mapping', level)) {
+            await this.pipelineLogService.debug(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                context: {
+                    adapterCode,
+                    inputFieldCount: inputFields.length,
+                    outputFieldCount: outputFields.length,
+                },
+                metadata: {
+                    inputFields,
+                    outputFields,
+                    mappings: mappings.slice(0, MAX_MAPPINGS_LOG),
+                },
+            });
+        }
+    }
+    /**
+     * Log a comparison between source and target for a record (DEBUG level)
+     * Records are automatically sanitized to remove sensitive data.
+     */
+    async logRecordTransformation(ctx, stepKey, recordIndex, sourceRecord, targetRecord, options) {
+        const message = `Record #${recordIndex} transformation in "${stepKey}"`;
+        // Always log to console at debug
+        this.consoleLogger.debug(message, {
+            stepKey,
+            recordIndex,
+            sourceFields: Object.keys(sourceRecord).length,
+            targetFields: Object.keys(targetRecord).length,
+        });
+        // Persist at DEBUG level (records are sanitized via truncateSample)
+        const level = await this.getPersistenceLevel();
+        if (this.shouldPersist('debug', level)) {
+            await this.pipelineLogService.debug(ctx, message, {
+                pipelineId: options.pipelineId,
+                runId: options.runId,
+                stepKey,
+                context: { recordIndex },
+                metadata: {
+                    source: this.truncateSample(sourceRecord),
+                    target: this.truncateSample(targetRecord),
+                },
+            });
+        }
+    }
+    /**
+     * Truncate and sanitize a sample record for safe logging.
+     * Removes sensitive fields and masks PII (emails, phone numbers).
+     */
+    truncateSample(record) {
+        // First sanitize to remove/mask sensitive data
+        const sanitized = (0, sanitizer_1.sanitizeRecord)(record);
+        // Then truncate long values
+        const result = {};
+        for (const [key, value] of Object.entries(sanitized)) {
+            result[key] = this.truncateValue(value);
+        }
+        return result;
+    }
+    /**
+     * Truncate a value for safe logging
+     */
+    truncateValue(value) {
+        if (value === null)
+            return null;
+        if (value === undefined)
+            return null;
+        if (typeof value === 'string') {
+            return value.length > index_1.TRUNCATION.MAX_FIELD_VALUE_LENGTH
+                ? value.substring(0, index_1.TRUNCATION.MAX_FIELD_VALUE_LENGTH) + '...'
+                : value;
+        }
+        if (typeof value === 'number' || typeof value === 'boolean') {
+            return value;
+        }
+        if (typeof value === 'object') {
+            try {
+                const str = JSON.stringify(value);
+                if (str.length > index_1.TRUNCATION.MAX_FIELD_VALUE_LENGTH) {
+                    return str.substring(0, index_1.TRUNCATION.MAX_FIELD_VALUE_LENGTH) + '...';
+                }
+                return value;
+            }
+            catch {
+                // JSON stringify failed (circular reference etc.) - return placeholder
+                return '[Object]';
+            }
+        }
+        return String(value);
+    }
+    /**
+     * Invalidate the cached persistence level (call when settings change)
+     */
+    invalidateCache() {
+        this.cachedLevel = null;
+        this.cacheTime = 0;
+    }
+    /**
+     * Get current persistence level (for external checks)
+     */
+    async getCurrentLevel() {
+        return this.getPersistenceLevel();
+    }
+};
+exports.ExecutionLogger = ExecutionLogger;
+exports.ExecutionLogger = ExecutionLogger = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [pipeline_log_service_1.PipelineLogService,
+        settings_service_1.DataHubSettingsService,
+        datahub_logger_1.DataHubLoggerFactory])
+], ExecutionLogger);
+//# sourceMappingURL=execution-logger.js.map