@open-mercato/core 0.6.7-develop.6744.1.0483b18565 → 0.6.7-develop.6749.1.6b54c56dfe

package/src/modules/data_sync/lib/sync-engine.ts

@@ -9,6 +9,7 @@ import { emitDataSyncEvent } from '../events'
 import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'
 import { getDataSyncAdapter } from './adapter-registry'
 import type { SyncRunService } from './sync-run-service'
+import { SyncRunOwnershipConflictError } from './sync-run-service'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
 const logger = createLogger('data_sync').child({ component: 'sync-engine' })
@@ -236,6 +237,21 @@ export function createSyncEngine(deps: EngineDeps) {
       return
     }
 
+    if (run.status !== status) {
+      // `markStatus` refuses a terminal -> different-terminal transition and
+      // returns the row unchanged, so the run is already finished under another
+      // delivery of this job. Everything below — the progress job, the
+      // operational log and the lifecycle event — would describe the wrong
+      // outcome, and `data_sync.run.failed` is dispatched to tenant webhooks.
+      // A displaced worker stays silent instead.
+      logger.warn('Skipping finalization of a sync run another worker already finalized', {
+        runId,
+        requestedStatus: status,
+        actualStatus: run.status,
+      })
+      return
+    }
+
     if (run.progressJobId) {
       if (status === 'completed') {
         await progressService.completeJob(
@@ -408,6 +424,10 @@ export function createSyncEngine(deps: EngineDeps) {
         throw new Error(`Integration ${run.integrationId} is missing credentials`)
       }
 
+      // A run already `running` means a stalled job was redelivered, so this is
+      // a resume rather than a first start. Consumers see one `started` event per
+      // delivery either way; the flag is what lets them tell the two apart.
+      const resumed = run.status === 'running'
       const activeRun = await syncRunService.markStatus(run.id, 'running', scope)
       if (!activeRun || activeRun.status !== 'running') {
         return
@@ -417,6 +437,7 @@ export function createSyncEngine(deps: EngineDeps) {
         integrationId: run.integrationId,
         entityType: run.entityType,
         direction: run.direction,
+        resumed,
         tenantId: scope.tenantId,
         organizationId: scope.organizationId,
       })
@@ -452,6 +473,7 @@ export function createSyncEngine(deps: EngineDeps) {
       const mapping = await resolveMapping(adapter, run.entityType, scope)
       let processedCount = 0
       let totalCount: number | null = null
+      let committedBatches = activeRun.batchesCompleted ?? 0
 
       try {
         for await (const batch of adapter.streamImport({
@@ -481,7 +503,9 @@ export function createSyncEngine(deps: EngineDeps) {
             },
             batch.cursor,
             scope,
+            committedBatches,
           )
+          committedBatches += 1
 
           await updateProgress(run.progressJobId, processedCount, totalCount, scope)
           await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)
@@ -507,6 +531,13 @@ export function createSyncEngine(deps: EngineDeps) {
           })
         }
       } catch (error) {
+        if (error instanceof SyncRunOwnershipConflictError) {
+          logger.warn('Yielding import run to a concurrent worker that already advanced it', {
+            runId: run.id,
+            expectedBatchesCompleted: error.expectedBatchesCompleted,
+          })
+          return
+        }
         const message = error instanceof Error ? error.message : 'Sync import failed'
         await integrationLogService.write(
           {
@@ -553,6 +584,10 @@ export function createSyncEngine(deps: EngineDeps) {
         throw new Error(`Integration ${run.integrationId} is missing credentials`)
       }
 
+      // A run already `running` means a stalled job was redelivered, so this is
+      // a resume rather than a first start. Consumers see one `started` event per
+      // delivery either way; the flag is what lets them tell the two apart.
+      const resumed = run.status === 'running'
       const activeRun = await syncRunService.markStatus(run.id, 'running', scope)
       if (!activeRun || activeRun.status !== 'running') {
         return
@@ -562,6 +597,7 @@ export function createSyncEngine(deps: EngineDeps) {
         integrationId: run.integrationId,
         entityType: run.entityType,
         direction: run.direction,
+        resumed,
         tenantId: scope.tenantId,
         organizationId: scope.organizationId,
       })
@@ -596,6 +632,7 @@ export function createSyncEngine(deps: EngineDeps) {
 
       const mapping = await resolveMapping(adapter, run.entityType, scope)
       let processedCount = 0
+      let committedBatches = activeRun.batchesCompleted ?? 0
 
       try {
         for await (const batch of adapter.streamExport({
@@ -626,7 +663,9 @@ export function createSyncEngine(deps: EngineDeps) {
             },
             batch.cursor,
             scope,
+            committedBatches,
           )
+          committedBatches += 1
           await updateProgress(run.progressJobId, processedCount, null, scope)
           await logExportItemFailures(run.id, run.integrationId, batch.results, scope)
 
@@ -647,6 +686,13 @@ export function createSyncEngine(deps: EngineDeps) {
           })
         }
       } catch (error) {
+        if (error instanceof SyncRunOwnershipConflictError) {
+          logger.warn('Yielding export run to a concurrent worker that already advanced it', {
+            runId: run.id,
+            expectedBatchesCompleted: error.expectedBatchesCompleted,
+          })
+          return
+        }
         const message = error instanceof Error ? error.message : 'Sync export failed'
         await integrationLogService.write(
           {

package/src/modules/data_sync/lib/sync-run-service.ts

@@ -26,6 +26,27 @@ type SyncScope = {
   tenantId: string
 }
 
+/**
+ * Raised when a batch commit loses the ownership compare-and-swap, meaning
+ * another delivery of the same job advanced the run while this worker was
+ * streaming.
+ *
+ * BullMQ guarantees at-least-once delivery: a job whose lock is not renewed is
+ * redelivered under the SAME job id, whether the previous worker died or is only
+ * blocked. No identity token can tell those apart, so ownership is enforced here
+ * — on the write that matters — instead of at claim time. The loser aborts and
+ * leaves the run to the worker that is still making progress.
+ */
+export class SyncRunOwnershipConflictError extends Error {
+  constructor(
+    readonly runId: string,
+    readonly expectedBatchesCompleted: number,
+  ) {
+    super(`[internal] Sync run ${runId} advanced past batch ${expectedBatchesCompleted} under a concurrent worker`)
+    this.name = 'SyncRunOwnershipConflictError'
+  }
+}
+
 export function createSyncRunService(em: EntityManager) {
   async function resolveCursorRow(run: SyncRun, scope: SyncScope): Promise<SyncCursor | null> {
     return findOneWithDecryption(
@@ -150,7 +171,11 @@ export function createSyncRunService(em: EntityManager) {
             organizationId: scope.organizationId,
             tenantId: scope.tenantId,
             deletedAt: null,
-            status: 'pending',
+            // A BullMQ stalled-job redelivery finds the run in `running` after
+            // the previous worker was hard-killed. Treat that transition as an
+            // idempotent claim while still excluding terminal states so a
+            // cancelled or completed run cannot be revived.
+            status: { $in: ['pending', 'running'] },
           },
           {
             status,
@@ -178,6 +203,12 @@ export function createSyncRunService(em: EntityManager) {
       return row
     },
 
+    /**
+     * @deprecated Use {@link commitBatchProgress}, which writes counters and
+     * cursor in one transaction behind the ownership fence. This method updates
+     * counters unfenced, so two deliveries of the same job can lose each other's
+     * increments. Kept for external callers only.
+     */
     async updateCounts(
       runId: string,
       delta: Partial<Pick<SyncRun, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount' | 'batchesCompleted'>>,
@@ -195,6 +226,11 @@ export function createSyncRunService(em: EntityManager) {
       return row
     },
 
+    /**
+     * @deprecated Use {@link commitBatchProgress}. This method advances the
+     * cursor without the ownership fence, so a stale delivery can move the
+     * cursor of a run another worker owns. Kept for external callers only.
+     */
     async updateCursor(runId: string, cursor: string, scope: SyncScope): Promise<void> {
       const run = await this.getRun(runId, scope)
       if (!run) return
@@ -204,16 +240,61 @@ export function createSyncRunService(em: EntityManager) {
       ], { transaction: true })
     },
 
+    /**
+     * Commits one batch's counters and cursor in a single transaction.
+     *
+     * Passing `expectedBatchesCompleted` fences the write: the run must still be
+     * `running` and still sit on that batch count, or another delivery of the
+     * same BullMQ job owns the run and this commit throws
+     * `SyncRunOwnershipConflictError` and rolls back. Omitting it keeps the
+     * legacy unguarded write for callers outside the engine.
+     *
+     * The fence token is `batchesCompleted` rather than `cursor` because it
+     * advances by construction on every commit. A cursor is a free-form adapter
+     * string that an adapter may legitimately repeat between batches — the
+     * Akeneo products adapter does, between its final page and the
+     * reconciliation batch that follows it — and a repeated token fences
+     * nothing.
+     *
+     * The guard's `UPDATE` also holds the row lock for the rest of the
+     * transaction, so a competing commit blocks here and then re-reads the
+     * advanced count instead of interleaving with this one. That is what lets
+     * the counters below stay a plain read-modify-write against the snapshot
+     * read above: a commit that wins the fence has proven that nothing else
+     * landed since it read.
+     */
     async commitBatchProgress(
       runId: string,
       delta: Partial<Pick<SyncRun, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount' | 'batchesCompleted'>>,
       cursor: string,
       scope: SyncScope,
+      expectedBatchesCompleted?: number,
     ): Promise<SyncRun | null> {
       const run = await this.getRun(runId, scope)
       if (!run) return null
       const cursorRow = await resolveCursorRow(run, scope)
+      const claimRunOwnership = async () => {
+        if ((delta.batchesCompleted ?? 0) < 1) {
+          throw new Error(`[internal] A fenced commit for sync run ${runId} must advance batchesCompleted`)
+        }
+        const owned = await em.nativeUpdate(
+          SyncRun,
+          {
+            id: runId,
+            organizationId: scope.organizationId,
+            tenantId: scope.tenantId,
+            deletedAt: null,
+            status: 'running',
+            batchesCompleted: expectedBatchesCompleted,
+          },
+          { updatedAt: new Date() },
+        )
+        if (owned === 0) {
+          throw new SyncRunOwnershipConflictError(runId, expectedBatchesCompleted ?? 0)
+        }
+      }
       await withAtomicFlush(em, [
+        ...(expectedBatchesCompleted === undefined ? [] : [claimRunOwnership]),
         () => {
           run.createdCount += delta.createdCount ?? 0
           run.updatedCount += delta.updatedCount ?? 0

package/src/modules/data_sync/workers/sync-export.ts

@@ -2,6 +2,11 @@ import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
 import type { ProgressService } from '../../progress/lib/progressService'
 import type { SyncEngine } from '../lib/sync-engine'
 import type { SyncRunService } from '../lib/sync-run-service'
+import {
+  DATA_SYNC_EXPORT_QUEUE,
+  DATA_SYNC_LOCK_DURATION_MS,
+  DATA_SYNC_MAX_STALLED_COUNT,
+} from '../lib/queue-policy'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
 const logger = createLogger('data_sync').child({ component: 'sync-export' })
@@ -17,9 +22,11 @@ type SyncJobPayload = {
 }
 
 export const metadata: WorkerMeta = {
-  queue: 'data-sync-export',
+  queue: DATA_SYNC_EXPORT_QUEUE,
   id: 'data-sync:export',
   concurrency: 5,
+  lockDuration: DATA_SYNC_LOCK_DURATION_MS,
+  maxStalledCount: DATA_SYNC_MAX_STALLED_COUNT,
 }
 
 type HandlerContext = JobContext & {

package/src/modules/data_sync/workers/sync-import.ts

@@ -2,6 +2,11 @@ import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
 import type { ProgressService } from '../../progress/lib/progressService'
 import type { SyncEngine } from '../lib/sync-engine'
 import type { SyncRunService } from '../lib/sync-run-service'
+import {
+  DATA_SYNC_IMPORT_QUEUE,
+  DATA_SYNC_LOCK_DURATION_MS,
+  DATA_SYNC_MAX_STALLED_COUNT,
+} from '../lib/queue-policy'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
 const logger = createLogger('data_sync').child({ component: 'sync-import' })
@@ -17,9 +22,11 @@ type SyncJobPayload = {
 }
 
 export const metadata: WorkerMeta = {
-  queue: 'data-sync-import',
+  queue: DATA_SYNC_IMPORT_QUEUE,
   id: 'data-sync:import',
   concurrency: 5,
+  lockDuration: DATA_SYNC_LOCK_DURATION_MS,
+  maxStalledCount: DATA_SYNC_MAX_STALLED_COUNT,
 }
 
 type HandlerContext = JobContext & {

package/src/modules/notifications/api/route.ts

@@ -3,6 +3,7 @@ import type { EntityManager } from '@mikro-orm/core'
 import { Notification } from '../data/entities'
 import { listNotificationsSchema, createNotificationSchema } from '../data/validators'
 import { toNotificationDto } from '../lib/notificationMapper'
+import { buildNotificationReadScopeWhere } from '../lib/notificationScope'
 import {
   NOTIFICATION_RESOURCE_KIND,
   notificationCrudErrorResponse,
@@ -32,6 +33,7 @@ export async function GET(req: Request) {
   const filters: Record<string, unknown> = {
     recipientUserId: scope.userId,
     tenantId: scope.tenantId,
+    ...buildNotificationReadScopeWhere(scope),
   }
 
   if (input.status) {

package/src/modules/notifications/api/unread-count/route.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto'
 import type { EntityManager } from '@mikro-orm/core'
 import { runWithCacheTenant } from '@open-mercato/cache'
 import {
@@ -8,6 +9,10 @@ import {
 import { Notification } from '../../data/entities'
 import { unreadCountResponseSchema } from '../openapi'
 import { resolveNotificationContext } from '../../lib/routeHelpers'
+import {
+  buildNotificationReadScopeWhere,
+  getNotificationReadScopeTagOrganizationIds,
+} from '../../lib/notificationScope'
 
 export const metadata = {
   GET: { requireAuth: true },
@@ -16,8 +21,21 @@ export const metadata = {
 const UNREAD_COUNT_RESOURCE = 'notifications.notification'
 const UNREAD_COUNT_TTL_MS = 10_000
 
-function buildUnreadCountCacheKey(userId: string): string {
-  return `notifications:unread-count:u=${userId}`
+function buildUnreadCountCacheKey(params: {
+  userId: string
+  organizationId: string | null
+  organizationIds: string[]
+}): string {
+  const normalizedIds = Array.from(new Set(
+    params.organizationIds.filter((value) => value.trim().length > 0),
+  )).sort((left, right) => left.localeCompare(right))
+  const scopeKey = normalizedIds.length === 0
+    ? 'no-access'
+    : `${params.organizationId ?? 'none'}:scope=${createHash('sha256')
+        .update(normalizedIds.join('\0'))
+        .digest('hex')
+        .slice(0, 16)}`
+  return `notifications:unread-count:u=${params.userId}:org=${scopeKey}`
 }
 
 export async function GET(req: Request) {
@@ -25,8 +43,22 @@ export async function GET(req: Request) {
   const em = ctx.container.resolve('em') as EntityManager
 
   const userId = scope.userId
-  const cache = userId && isCrudCacheEnabled() ? resolveCrudCache(ctx.container) : null
-  const cacheKey = cache && userId ? buildUnreadCountCacheKey(userId) : null
+  const cacheableOrganizationIds = Array.isArray(scope.organizationIds)
+    ? scope.organizationIds
+    : null
+  // Unrestricted and omitted legacy scopes cannot be safely tagged for invalidation:
+  // org-specific writes only invalidate their own collection tag. Keep those scopes
+  // uncached rather than serving a stale tenant-wide total until the TTL expires.
+  const cache = userId && cacheableOrganizationIds && isCrudCacheEnabled()
+    ? resolveCrudCache(ctx.container)
+    : null
+  const cacheKey = cache && userId && cacheableOrganizationIds
+    ? buildUnreadCountCacheKey({
+        userId,
+        organizationId: scope.organizationId,
+        organizationIds: cacheableOrganizationIds,
+      })
+    : null
 
   if (cache && cacheKey) {
     try {
@@ -41,6 +73,7 @@ export async function GET(req: Request) {
     recipientUserId: userId,
     tenantId: scope.tenantId,
     status: 'unread',
+    ...buildNotificationReadScopeWhere(scope),
   })
 
   if (cache && cacheKey) {
@@ -48,7 +81,11 @@ export async function GET(req: Request) {
       await runWithCacheTenant(scope.tenantId, () =>
         cache.set(cacheKey, count, {
           ttl: UNREAD_COUNT_TTL_MS,
-          tags: buildCollectionTags(UNREAD_COUNT_RESOURCE, scope.tenantId, [null]),
+          tags: buildCollectionTags(
+            UNREAD_COUNT_RESOURCE,
+            scope.tenantId,
+            getNotificationReadScopeTagOrganizationIds(scope),
+          ),
         }),
       )
     } catch {}

package/src/modules/notifications/lib/notificationScope.ts

@@ -0,0 +1,40 @@
+export type NotificationReadOrganizationScope = {
+  organizationId?: string | null
+  organizationIds?: string[] | null
+}
+
+type NotificationReadScopeWhere = {
+  organizationId?: null
+  $or?: Array<
+    | { organizationId: { $in: string[] } }
+    | { organizationId: null }
+  >
+}
+
+function resolveOrganizationIds(scope: NotificationReadOrganizationScope): string[] | null {
+  if (scope.organizationIds === null) return null
+  if (scope.organizationIds === undefined) return null
+  const candidates = scope.organizationIds
+  return Array.from(new Set(candidates.filter((value) => value.trim().length > 0)))
+}
+
+export function buildNotificationReadScopeWhere(
+  scope: NotificationReadOrganizationScope,
+): NotificationReadScopeWhere {
+  const organizationIds = resolveOrganizationIds(scope)
+  if (organizationIds === null) return {}
+  if (organizationIds.length === 0) return { organizationId: null }
+  return {
+    $or: [
+      { organizationId: { $in: organizationIds } },
+      { organizationId: null },
+    ],
+  }
+}
+
+export function getNotificationReadScopeTagOrganizationIds(
+  scope: NotificationReadOrganizationScope,
+): Array<string | null> {
+  const organizationIds = resolveOrganizationIds(scope)
+  return organizationIds === null ? [null] : [...organizationIds, null]
+}

package/src/modules/notifications/lib/notificationService.ts

@@ -14,6 +14,7 @@ import {
   type NotificationTenantContext,
 } from './notificationFactory'
 import { toNotificationDto } from './notificationMapper'
+import { buildNotificationReadScopeWhere } from './notificationScope'
 import {
   getRecipientUserIdsForFeature,
   getRecipientUserIdsForRole,
@@ -184,6 +185,7 @@ async function createOrRefreshNotification(
 export interface NotificationServiceContext {
   tenantId: string
   organizationId?: string | null
+  organizationIds?: string[] | null
   userId?: string | null
 }
 
@@ -594,6 +596,7 @@ export function createNotificationService(deps: NotificationServiceDeps): Notifi
         recipientUserId: ctx.userId,
         tenantId: ctx.tenantId,
         status: 'unread',
+        ...buildNotificationReadScopeWhere(ctx),
       })
     },
 
@@ -602,6 +605,7 @@ export function createNotificationService(deps: NotificationServiceDeps): Notifi
       const filters: Record<string, unknown> = {
         recipientUserId: ctx.userId,
         tenantId: ctx.tenantId,
+        ...buildNotificationReadScopeWhere(ctx),
       }
 
       if (since) {
@@ -617,6 +621,7 @@ export function createNotificationService(deps: NotificationServiceDeps): Notifi
           recipientUserId: ctx.userId,
           tenantId: ctx.tenantId,
           status: 'unread',
+          ...buildNotificationReadScopeWhere(ctx),
         }),
       ])
 

package/src/modules/notifications/lib/routeHelpers.ts

@@ -4,6 +4,7 @@ import { resolveRequestContext } from '@open-mercato/shared/lib/api/context'
 import { isCrudHttpError } from '@open-mercato/shared/lib/crud/errors'
 import { runRouteMutationGuards } from '@open-mercato/shared/lib/crud/route-mutation-guard'
 import { resolveTranslations } from '@open-mercato/shared/lib/i18n/server'
+import { resolveOrganizationScopeForRequest } from '@open-mercato/core/modules/directory/utils/organizationScope'
 import { resolveNotificationService, type NotificationService } from './notificationService'
 
 /**
@@ -22,6 +23,7 @@ export const NOTIFICATION_SETTINGS_RESOURCE_KIND = 'notifications.settings'
 export interface NotificationScope {
   tenantId: string
   organizationId: string | null
+  organizationIds?: string[] | null
   userId: string | null
 }
 
@@ -64,11 +66,26 @@ export function notificationCrudErrorResponse(error: unknown): Response | null {
  */
 export async function resolveNotificationContext(req: Request): Promise<NotificationRequestContext> {
   const { ctx } = await resolveRequestContext(req)
+  const organizationScope = await resolveOrganizationScopeForRequest({
+    container: ctx.container,
+    auth: ctx.auth,
+    request: req,
+    ...(ctx.selectedOrganizationId === undefined
+      ? {}
+      : { selectedId: ctx.selectedOrganizationId }),
+  })
+  const tenantId = organizationScope.tenantId ?? ctx.auth?.tenantId ?? ''
+  const organizationId = organizationScope.selectedId
+  const organizationIds = organizationScope.filterIds
+  ctx.organizationScope = organizationScope
+  ctx.selectedOrganizationId = organizationId
+  ctx.organizationIds = organizationIds
   return {
     service: resolveNotificationService(ctx.container),
     scope: {
-      tenantId: ctx.auth?.tenantId ?? '',
-      organizationId: ctx.selectedOrganizationId ?? null,
+      tenantId,
+      organizationId,
+      organizationIds,
       userId: ctx.auth?.sub ?? null,
     },
     ctx,