@open-mercato/core 0.6.2-develop.3330.1.8ad2b3b84c → 0.6.2-develop.3366.1.4692954e44

package/src/modules/data_sync/lib/sync-engine.ts

@@ -2,6 +2,7 @@ import type { EntityManager } from '@mikro-orm/postgresql'
 import { getIntegration } from '@open-mercato/shared/modules/integrations/types'
 import type { CredentialsService } from '../../integrations/lib/credentials-service'
 import type { IntegrationLogService } from '../../integrations/lib/log-service'
+import type { IntegrationStateService } from '../../integrations/lib/state-service'
 import type { ProgressService } from '../../progress/lib/progressService'
 import { refreshCoverageSnapshot } from '../../query_index/lib/coverage'
 import { emitDataSyncEvent } from '../events'
@@ -20,6 +21,7 @@ type EngineDeps = {
   syncRunService: SyncRunService
   integrationCredentialsService: CredentialsService
   integrationLogService: IntegrationLogService
+  integrationStateService?: IntegrationStateService
   progressService: ProgressService
 }
 
@@ -71,7 +73,7 @@ function applyExportCounters(batch: ExportBatch): SyncCounterDelta {
 }
 
 export function createSyncEngine(deps: EngineDeps) {
-  const { syncRunService, integrationCredentialsService, integrationLogService, progressService } = deps
+  const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps
 
   async function resolveMapping(adapter: DataSyncAdapter, entityType: string, scope: SyncScope): Promise<DataMapping> {
     return adapter.getMapping({
@@ -128,7 +130,7 @@ export function createSyncEngine(deps: EngineDeps) {
         ? item.data.sourceIdentifier.trim()
         : null
       const message = [
-        `Failed to import Akeneo product ${item.externalId}`,
+        `Failed to import item ${item.externalId}`,
         sourceProductUuid ? `(uuid: ${sourceProductUuid})` : null,
         sourceIdentifier ? `(identifier: ${sourceIdentifier})` : null,
         `: ${errorMessage}`,
@@ -147,10 +149,65 @@ export function createSyncEngine(deps: EngineDeps) {
     }
   }
 
-  async function finalizeRun(runId: string, status: 'completed' | 'failed' | 'cancelled', scope: SyncScope, error?: string): Promise<void> {
+  async function writeOperationalLog(params: {
+    integrationId: string
+    runId: string
+    level: 'info' | 'warn' | 'error'
+    message: string
+    scope: SyncScope
+    enabled: boolean
+    payload?: Record<string, unknown>
+  }): Promise<void> {
+    if (!params.enabled) return
+
+    await integrationLogService.write(
+      {
+        integrationId: params.integrationId,
+        runId: params.runId,
+        level: params.level,
+        message: params.message,
+        payload: params.payload,
+      },
+      params.scope,
+    )
+  }
+
+  async function updateOperationalState(params: {
+    integrationId: string
+    status: 'healthy' | 'degraded' | 'unhealthy'
+    scope: SyncScope
+    enabled: boolean
+  }): Promise<void> {
+    if (!params.enabled || !integrationStateService) return
+
+    await integrationStateService.upsert(
+      params.integrationId,
+      {
+        lastHealthStatus: params.status,
+        lastHealthCheckedAt: new Date(),
+      },
+      params.scope,
+    )
+  }
+
+  async function finalizeRun(
+    runId: string,
+    status: 'completed' | 'failed' | 'cancelled',
+    scope: SyncScope,
+    error?: string,
+    operationalTelemetry = false,
+  ): Promise<void> {
+    const existingRun = await syncRunService.getRun(runId, scope)
+    const alreadyFinalizedWithSameStatus = existingRun?.status === status
+      && (status === 'completed' || status === 'failed' || status === 'cancelled')
+
     const run = await syncRunService.markStatus(runId, status, scope, error)
     if (!run) return
 
+    if (alreadyFinalizedWithSameStatus) {
+      return
+    }
+
     if (run.progressJobId) {
       if (status === 'completed') {
         await progressService.completeJob(
@@ -194,6 +251,70 @@ export function createSyncEngine(deps: EngineDeps) {
       }
     }
 
+    if (status === 'completed') {
+      await updateOperationalState({
+        integrationId: run.integrationId,
+        status: 'healthy',
+        scope,
+        enabled: operationalTelemetry,
+      })
+      await writeOperationalLog({
+        integrationId: run.integrationId,
+        runId: run.id,
+        level: 'info',
+        message: 'Sync run completed',
+        scope,
+        enabled: operationalTelemetry,
+        payload: {
+          operationalStatus: 'completed',
+          summary: `Sync completed with ${run.createdCount} created, ${run.updatedCount} updated, ${run.failedCount} failed.`,
+          createdCount: run.createdCount,
+          updatedCount: run.updatedCount,
+          skippedCount: run.skippedCount,
+          failedCount: run.failedCount,
+          batchesCompleted: run.batchesCompleted,
+        },
+      })
+    } else if (status === 'cancelled') {
+      await updateOperationalState({
+        integrationId: run.integrationId,
+        status: 'degraded',
+        scope,
+        enabled: operationalTelemetry,
+      })
+      await writeOperationalLog({
+        integrationId: run.integrationId,
+        runId: run.id,
+        level: 'warn',
+        message: 'Sync run cancelled',
+        scope,
+        enabled: operationalTelemetry,
+        payload: {
+          operationalStatus: 'cancelled',
+          summary: 'The sync run was cancelled before completion.',
+        },
+      })
+    } else {
+      await updateOperationalState({
+        integrationId: run.integrationId,
+        status: 'unhealthy',
+        scope,
+        enabled: operationalTelemetry,
+      })
+      await writeOperationalLog({
+        integrationId: run.integrationId,
+        runId: run.id,
+        level: 'error',
+        message: error ?? 'Sync run failed',
+        scope,
+        enabled: operationalTelemetry,
+        payload: {
+          operationalStatus: 'failed',
+          summary: error ?? 'The sync run failed.',
+        },
+      })
+    }
+
     if (status === 'completed') {
       await emitDataSyncEvent('data_sync.run.completed', {
         runId,
@@ -252,6 +373,7 @@ export function createSyncEngine(deps: EngineDeps) {
       if (!adapter?.streamImport) {
         throw new Error(`No import adapter registered for provider ${providerKey}`)
       }
+      const operationalTelemetry = adapter.operationalTelemetry === true
 
       const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)
       if (!credentials) {
@@ -260,13 +382,6 @@ export function createSyncEngine(deps: EngineDeps) {
 
       const activeRun = await syncRunService.markStatus(run.id, 'running', scope)
       if (!activeRun || activeRun.status !== 'running') {
-        if (run.progressJobId) {
-          await progressService.markCancelled(run.progressJobId, {
-            tenantId: scope.tenantId,
-            organizationId: scope.organizationId,
-            userId: scope.userId,
-          })
-        }
         return
       }
       await emitDataSyncEvent('data_sync.run.started', {
@@ -277,6 +392,26 @@ export function createSyncEngine(deps: EngineDeps) {
         tenantId: scope.tenantId,
         organizationId: scope.organizationId,
       })
+      await updateOperationalState({
+        integrationId: run.integrationId,
+        status: 'degraded',
+        scope,
+        enabled: operationalTelemetry,
+      })
+      await writeOperationalLog({
+        integrationId: run.integrationId,
+        runId: run.id,
+        level: 'info',
+        message: 'Sync run started',
+        scope,
+        enabled: operationalTelemetry,
+        payload: {
+          operationalStatus: 'running',
+          summary: `Import run started for ${run.entityType}.`,
+          entityType: run.entityType,
+          direction: run.direction,
+        },
+      })
 
       if (run.progressJobId) {
         await progressService.startJob(run.progressJobId, {
@@ -298,9 +433,10 @@ export function createSyncEngine(deps: EngineDeps) {
           credentials,
           mapping,
           scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
+          runId: run.id,
         })) {
           if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId)) {
-            await finalizeRun(run.id, 'cancelled', scope)
+            await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
             return
           }
 
@@ -323,23 +459,24 @@ export function createSyncEngine(deps: EngineDeps) {
           await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)
           await logImportItemFailures(run.id, run.integrationId, batch.items, scope)
 
-          await integrationLogService.write(
-            {
-              integrationId: run.integrationId,
-              runId: run.id,
-              level: 'info',
-              message: batch.message?.trim().length
-                ? batch.message.trim()
-                : `Processed import batch ${batch.batchIndex}`,
-              payload: {
-                processedCount,
-                batchSize: batch.items.length,
-                processedBatchCount,
-                cursor: batch.cursor,
-              },
-            },
+          await writeOperationalLog({
+            integrationId: run.integrationId,
+            runId: run.id,
+            level: 'info',
+            message: batch.message?.trim().length
+              ? batch.message.trim()
+              : `Processed import batch ${batch.batchIndex}`,
             scope,
-          )
+            enabled: operationalTelemetry,
+            payload: {
+              operationalStatus: 'running',
+              summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,
+              processedCount,
+              batchSize: batch.items.length,
+              processedBatchCount,
+              cursor: batch.cursor,
+            },
+          })
         }
       } catch (error) {
         const message = error instanceof Error ? error.message : 'Sync import failed'
@@ -352,11 +489,11 @@ export function createSyncEngine(deps: EngineDeps) {
           },
           scope,
         )
-        await finalizeRun(run.id, 'failed', scope, message)
+        await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)
         return
       }
 
-      await finalizeRun(run.id, 'completed', scope)
+      await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)
     },
 
     async runExport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {
@@ -381,6 +518,7 @@ export function createSyncEngine(deps: EngineDeps) {
       if (!adapter?.streamExport) {
         throw new Error(`No export adapter registered for provider ${providerKey}`)
       }
+      const operationalTelemetry = adapter.operationalTelemetry === true
 
       const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)
       if (!credentials) {
@@ -389,13 +527,6 @@ export function createSyncEngine(deps: EngineDeps) {
 
       const activeRun = await syncRunService.markStatus(run.id, 'running', scope)
       if (!activeRun || activeRun.status !== 'running') {
-        if (run.progressJobId) {
-          await progressService.markCancelled(run.progressJobId, {
-            tenantId: scope.tenantId,
-            organizationId: scope.organizationId,
-            userId: scope.userId,
-          })
-        }
         return
       }
       await emitDataSyncEvent('data_sync.run.started', {
@@ -406,6 +537,26 @@ export function createSyncEngine(deps: EngineDeps) {
         tenantId: scope.tenantId,
         organizationId: scope.organizationId,
       })
+      await updateOperationalState({
+        integrationId: run.integrationId,
+        status: 'degraded',
+        scope,
+        enabled: operationalTelemetry,
+      })
+      await writeOperationalLog({
+        integrationId: run.integrationId,
+        runId: run.id,
+        level: 'info',
+        message: 'Sync run started',
+        scope,
+        enabled: operationalTelemetry,
+        payload: {
+          operationalStatus: 'running',
+          summary: `Export run started for ${run.entityType}.`,
+          entityType: run.entityType,
+          direction: run.direction,
+        },
+      })
 
       if (run.progressJobId) {
         await progressService.startJob(run.progressJobId, {
@@ -426,9 +577,10 @@ export function createSyncEngine(deps: EngineDeps) {
           credentials,
           mapping,
           scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
+          runId: run.id,
         })) {
           if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId)) {
-            await finalizeRun(run.id, 'cancelled', scope)
+            await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
             return
           }
 
@@ -450,20 +602,21 @@ export function createSyncEngine(deps: EngineDeps) {
           await syncRunService.updateCursor(run.id, batch.cursor, scope)
           await updateProgress(run.progressJobId, processedCount, null, scope)
 
-          await integrationLogService.write(
-            {
-              integrationId: run.integrationId,
-              runId: run.id,
-              level: 'info',
-              message: `Processed export batch ${batch.batchIndex}`,
-              payload: {
-                processedCount,
-                batchSize: batch.results.length,
-                cursor: batch.cursor,
-              },
-            },
+          await writeOperationalLog({
+            integrationId: run.integrationId,
+            runId: run.id,
+            level: 'info',
+            message: `Processed export batch ${batch.batchIndex}`,
             scope,
-          )
+            enabled: operationalTelemetry,
+            payload: {
+              operationalStatus: 'running',
+              summary: `Processed ${processedCount} export items so far.`,
+              processedCount,
+              batchSize: batch.results.length,
+              cursor: batch.cursor,
+            },
+          })
         }
       } catch (error) {
         const message = error instanceof Error ? error.message : 'Sync export failed'
@@ -476,11 +629,11 @@ export function createSyncEngine(deps: EngineDeps) {
           },
           scope,
         )
-        await finalizeRun(run.id, 'failed', scope, message)
+        await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)
         return
       }
 
-      await finalizeRun(run.id, 'completed', scope)
+      await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)
     },
   }
 }

package/src/modules/data_sync/lib/sync-run-service.ts

@@ -86,6 +86,30 @@ export function createSyncRunService(em: EntityManager) {
     },
 
     async markStatus(runId: string, status: SyncRun['status'], scope: SyncScope, error?: string): Promise<SyncRun | null> {
+      if (status === 'running') {
+        const updated = await em.nativeUpdate(
+          SyncRun,
+          {
+            id: runId,
+            organizationId: scope.organizationId,
+            tenantId: scope.tenantId,
+            deletedAt: null,
+            status: 'pending',
+          },
+          {
+            status,
+            ...(error !== undefined ? { lastError: error } : {}),
+            updatedAt: new Date(),
+          },
+        )
+        if (updated === 0) return null
+        const row = await this.getRun(runId, scope)
+        if (row && typeof em.refresh === 'function') {
+          await em.refresh(row)
+        }
+        return row
+      }
+
       const row = await this.getRun(runId, scope)
       if (!row) return null
       const isTerminal = row.status === 'completed' || row.status === 'failed' || row.status === 'cancelled'