@open-mercato/core 0.6.8-develop.6986.1.3adb0d0df6 → 0.6.8-develop.6992.1.00c90fecff

package/src/modules/data_sync/lib/sync-run-service.ts

@@ -26,6 +26,25 @@ type SyncScope = {
   tenantId: string
 }
 
+export type CursorCommitOptions = {
+  /**
+   * Mirror the committed cursor into the shared `sync_cursors` row. Defaults to
+   * `true`; the engine passes the adapter's `persistsSharedCursor(entityType)`
+   * verdict. `false` keeps the cursor on the run row alone.
+   */
+  persistSharedCursor?: boolean
+  /**
+   * Fences the write against a concurrent delivery: the run must still be
+   * `running` and still sit on this batch count, or the commit throws
+   * {@link SyncRunOwnershipConflictError} and rolls back. Omit to keep the
+   * unguarded write for callers outside the engine.
+   */
+  expectedBatchesCompleted?: number
+}
+
+/** {@link CursorCommitOptions} minus the fence, which `updateCursor` does not apply. */
+export type SharedCursorOption = Pick<CursorCommitOptions, 'persistSharedCursor'>
+
 /**
  * Raised when a batch commit loses the ownership compare-and-swap, meaning
  * another delivery of the same job advanced the run while this worker was
@@ -64,8 +83,15 @@ export function createSyncRunService(em: EntityManager) {
     )
   }
 
-  function applyCursorMutation(run: SyncRun, cursorRow: SyncCursor | null, cursor: string, scope: SyncScope): void {
+  function applyCursorMutation(
+    run: SyncRun,
+    cursorRow: SyncCursor | null,
+    cursor: string,
+    scope: SyncScope,
+    persistSharedCursor: boolean,
+  ): void {
     run.cursor = cursor
+    if (!persistSharedCursor) return
     if (cursorRow) {
       cursorRow.cursor = cursor
     } else {
@@ -230,25 +256,37 @@ export function createSyncRunService(em: EntityManager) {
      * @deprecated Use {@link commitBatchProgress}. This method advances the
      * cursor without the ownership fence, so a stale delivery can move the
      * cursor of a run another worker owns. Kept for external callers only.
+     *
+     * It still takes `persistSharedCursor` despite being deprecated: an external
+     * caller advancing the cursor of an opted-out entity type would otherwise
+     * create the very `sync_cursors` row the opt-out exists to avoid, and a
+     * later incremental run would read it as a start position. The deprecated
+     * path has to honour the opt-out for as long as it exists.
      */
-    async updateCursor(runId: string, cursor: string, scope: SyncScope): Promise<void> {
+    async updateCursor(runId: string, cursor: string, scope: SyncScope, options?: SharedCursorOption): Promise<void> {
       const run = await this.getRun(runId, scope)
       if (!run) return
-      const cursorRow = await resolveCursorRow(run, scope)
+      const persistSharedCursor = options?.persistSharedCursor ?? true
+      const cursorRow = persistSharedCursor ? await resolveCursorRow(run, scope) : null
       await withAtomicFlush(em, [
-        () => applyCursorMutation(run, cursorRow, cursor, scope),
+        () => applyCursorMutation(run, cursorRow, cursor, scope, persistSharedCursor),
       ], { transaction: true })
     },
 
     /**
      * Commits one batch's counters and cursor in a single transaction.
      *
-     * Passing `expectedBatchesCompleted` fences the write: the run must still be
-     * `running` and still sit on that batch count, or another delivery of the
-     * same BullMQ job owns the run and this commit throws
+     * Passing `options.expectedBatchesCompleted` fences the write: the run must
+     * still be `running` and still sit on that batch count, or another delivery
+     * of the same BullMQ job owns the run and this commit throws
      * `SyncRunOwnershipConflictError` and rolls back. Omitting it keeps the
      * legacy unguarded write for callers outside the engine.
      *
+     * `options.persistSharedCursor` is orthogonal to the fence: it decides
+     * whether the committed cursor is mirrored into the shared `sync_cursors`
+     * row, and the two compose freely — a fenced commit for an opted-out entity
+     * type advances the run row alone and still throws on a stale fence.
+     *
      * The fence token is `batchesCompleted` rather than `cursor` because it
      * advances by construction on every commit. A cursor is a free-form adapter
      * string that an adapter may legitimately repeat between batches — the
@@ -268,11 +306,13 @@ export function createSyncRunService(em: EntityManager) {
       delta: Partial<Pick<SyncRun, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount' | 'batchesCompleted'>>,
       cursor: string,
       scope: SyncScope,
-      expectedBatchesCompleted?: number,
+      options?: CursorCommitOptions,
     ): Promise<SyncRun | null> {
       const run = await this.getRun(runId, scope)
       if (!run) return null
-      const cursorRow = await resolveCursorRow(run, scope)
+      const { expectedBatchesCompleted } = options ?? {}
+      const persistSharedCursor = options?.persistSharedCursor ?? true
+      const cursorRow = persistSharedCursor ? await resolveCursorRow(run, scope) : null
       const claimRunOwnership = async () => {
         if ((delta.batchesCompleted ?? 0) < 1) {
           throw new Error(`[internal] A fenced commit for sync run ${runId} must advance batchesCompleted`)
@@ -301,7 +341,7 @@ export function createSyncRunService(em: EntityManager) {
           run.skippedCount += delta.skippedCount ?? 0
           run.failedCount += delta.failedCount ?? 0
           run.batchesCompleted += delta.batchesCompleted ?? 0
-          applyCursorMutation(run, cursorRow, cursor, scope)
+          applyCursorMutation(run, cursorRow, cursor, scope, persistSharedCursor)
         },
       ], { transaction: true })
       return run
@@ -324,6 +364,74 @@ export function createSyncRunService(em: EntityManager) {
       return row?.cursor ?? null
     },
 
+    /**
+     * Resume position for an entity type whose adapter opted out of the shared
+     * `sync_cursors` row: the cursor of the most recent run, unless that run
+     * reached `completed`. A finished walk resumes from `null` so the next run
+     * starts over rather than skipping everything an older interrupted run had
+     * already passed.
+     */
+    async resolveResumeCursor(integrationId: string, entityType: string, direction: 'import' | 'export', scope: SyncScope): Promise<string | null> {
+      const [run] = await findWithDecryption(
+        em,
+        SyncRun,
+        {
+          integrationId,
+          entityType,
+          direction,
+          organizationId: scope.organizationId,
+          tenantId: scope.tenantId,
+          deletedAt: null,
+        },
+        { orderBy: { createdAt: 'DESC' }, limit: 1 },
+        scope,
+      )
+      if (!run || run.status === 'completed') return null
+      return run.cursor ?? null
+    },
+
+    /**
+     * Clears the run-scoped resume position for an entity type, so the next
+     * non-`fullSync` run starts from the beginning. Returns how many runs were
+     * cleared.
+     *
+     * This is the opt-out's equivalent of deleting the shared `sync_cursors`
+     * row. An entity type whose adapter returns `false` from
+     * `persistsSharedCursor` has no such row, so a reset flow that only deletes
+     * `SyncCursor` would leave {@link resolveResumeCursor} returning the cursor
+     * of the interrupted run it just reset against — re-importing the tail of a
+     * walk instead of the whole thing. Reset flows MUST call this alongside
+     * their `SyncCursor` delete; it is a no-op when nothing is interrupted.
+     *
+     * The `status` filter here selects which rows to clear. It is deliberately
+     * NOT the read-side filter {@link resolveResumeCursor} avoids: that method
+     * reads the single most recent run whatever its status, precisely so an
+     * older interrupted run cannot outlive a later completed walk. Clearing
+     * every interrupted run is enough to start fresh either way — if the latest
+     * run was interrupted its cursor is now null, and if it completed the resume
+     * path already returns null.
+     */
+    async resetResumePosition(
+      integrationId: string,
+      entityType: string,
+      direction: 'import' | 'export',
+      scope: SyncScope,
+    ): Promise<number> {
+      return em.nativeUpdate(
+        SyncRun,
+        {
+          integrationId,
+          entityType,
+          direction,
+          status: { $ne: 'completed' },
+          organizationId: scope.organizationId,
+          tenantId: scope.tenantId,
+          deletedAt: null,
+        },
+        { cursor: null, updatedAt: new Date() },
+      )
+    },
+
     async findRunningOverlap(integrationId: string, entityType: string, direction: 'import' | 'export', scope: SyncScope): Promise<SyncRun | null> {
       const [run] = await findWithDecryption(
         em,

package/src/modules/data_sync/workers/sync-scheduled.ts

@@ -6,6 +6,7 @@ import type { ProgressService } from '../../progress/lib/progressService'
 import type { SyncRunService } from '../lib/sync-run-service'
 import { SyncSchedule } from '../data/entities'
 import { startDataSyncRun } from '../lib/start-run'
+import { resolveAdapterForIntegration, resolveStartCursor } from '../lib/start-cursor'
 
 type ScheduledSyncPayload = {
   scheduleId: string
@@ -65,12 +66,14 @@ export default async function handle(job: QueuedJob<ScheduledSyncPayload>, ctx:
 
   const cursor = schedule.fullSync
     ? null
-    : await syncRunService.resolveCursor(
-        schedule.integrationId,
-        schedule.entityType,
-        schedule.direction,
-        job.payload.scope,
-      )
+    : await resolveStartCursor({
+        syncRunService,
+        adapter: resolveAdapterForIntegration(schedule.integrationId),
+        integrationId: schedule.integrationId,
+        entityType: schedule.entityType,
+        direction: schedule.direction,
+        scope: job.payload.scope,
+      })
 
   schedule.lastRunAt = new Date()
   await em.flush()

package/src/modules/progress/AGENTS.md

@@ -93,7 +93,8 @@ Use stable, grep-friendly ids:
 `progressService` is safe to run across many app/worker instances. Do not reintroduce read-modify-write transitions:
 
 - **Every status transition is a status-guarded `nativeUpdate` (CAS).** An update that matches zero rows lost the race — it MUST NOT emit events or overwrite the row. Allowed transitions: start from `pending|failed` (an already-running start is an idempotent no-op; `failed` allows queue retries and recovery from a wrong stale sweep), complete from `pending|running|failed`, fail from `pending|running`, cancel from `pending|running|failed`.
-- **Progress writes are guarded on `status IN ('pending','running')`** — once another process finishes/cancels a job, buffered updaters stop writing to it.
+- **Progress writes are guarded on `status IN ('pending','running')`** — once another process finishes/cancels a job, buffered updaters stop writing to it. Exception: when a write path finds the job `failed`, it attempts exactly one revival through the start CAS (a live producer writing progress proves a stale sweep false-positived) before giving up. That revive CAS is narrowed to rows whose `errorMessage` starts with `STALE_SWEEP_ERROR_PREFIX`, so a job that failed for a real reason keeps its message and stack instead of being resurrected by a racing buffered writer; `completed`/`cancelled` are never revived, and genuinely dead producers never write again, so real sweeps stick. Only `startJob` uses the unrestricted CAS, because a queue retry restarts the whole unit of work. A revive never resets `startedAt` — progress counts are absolute across deliveries, so the elapsed window `calculateEta` divides them by must be too.
+- **Long single units of work MUST call `touchJobHeartbeat`** — when one batch/step can outlast `STALE_JOB_TIMEOUT_SECONDS` (60s), run a keepalive that calls the optional `touchJobHeartbeat(jobId, ctx)` while awaiting it (see `withHeartbeat` in `data_sync/lib/sync-engine.ts`). It writes only `heartbeat_at`/`updated_at` on a **forked** EntityManager, so it is safe while the producer's own EM is mid-transaction, and it self-heals a falsely-swept job via the start CAS. Always optional-chain it (`progressService.touchJobHeartbeat?.(...)`) — third-party implementations may not provide it. The "no timers for durable work" rule below bans replacing queue delivery with timers, not an in-process keepalive around a pending await.
 - **`incrementProgress` deltas persist as atomic SQL increments** (`processed_count + n`), and the service reloads the database winner before returning or emitting, so concurrent writers never lose or report stale counts.
 - **Update-path reads use `disableIdentityMap: true`** — `isCancellationRequested` and lifecycle reads must always see fresh cross-process state, never a stale managed entity.
 - **The stale sweep (`markStaleJobsFailed`) re-checks staleness per row inside the CAS**, so concurrent sweepers emit exactly one `JOB_FAILED` per job, and it also fails `pending` jobs that never started within `STALE_PENDING_TIMEOUT_SECONDS` (a late queue delivery recovers them via `startJob`'s `failed → running` transition).

package/src/modules/progress/lib/progressService.ts

@@ -21,12 +21,21 @@ export interface ProgressService {
   getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>
   getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>
   markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>
+  // Optional so third-party ProgressService implementations keep compiling; callers must
+  // optional-chain. Runs on a forked EntityManager, so it is safe to call while the shared
+  // request/worker EM is mid-transaction (e.g. from a keepalive timer around adapter I/O).
+  touchJobHeartbeat?(jobId: string, ctx: ProgressServiceContext): Promise<void>
 }
 
 export const HEARTBEAT_INTERVAL_MS = 5000
 export const STALE_JOB_TIMEOUT_SECONDS = 60
 export const STALE_PENDING_TIMEOUT_SECONDS = 900
 
+// Every `errorMessage` the stale sweep writes starts with this, so recovery paths can tell
+// a job the sweep gave up on from one that failed for a real reason and must keep its
+// diagnostics. Declared here so the sweep and the revive filter cannot drift apart.
+export const STALE_SWEEP_ERROR_PREFIX = 'Job stale:'
+
 export function calculateEta(
   processedCount: number,
   totalCount: number,

package/src/modules/progress/lib/progressServiceImpl.ts

@@ -9,6 +9,7 @@ import {
   HEARTBEAT_INTERVAL_MS,
   STALE_JOB_TIMEOUT_SECONDS,
   STALE_PENDING_TIMEOUT_SECONDS,
+  STALE_SWEEP_ERROR_PREFIX,
 } from './progressService'
 import { PROGRESS_EVENTS } from './events'
 import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
@@ -131,6 +132,58 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
     return {}
   }
 
+  // The start transition (START_FROM_STATUSES) doubles as the recovery path for jobs a
+  // stale sweep flipped to `failed` while their producer was alive but slow. Callers pass
+  // the EM to run against so the forked heartbeat path can reuse the same CAS.
+  //
+  // `staleSweptOnly` is for the revive paths: a live write proves the producer is alive,
+  // but it does NOT prove the recorded failure was the sweep's. Narrowing the CAS to rows
+  // the sweep tagged keeps a genuine failure's message and stack intact instead of
+  // resurrecting the job and erasing why it died. `startJob` stays unrestricted so an
+  // at-least-once queue can still retry a genuinely failed job from the top.
+  async function startJobViaCas(
+    targetEm: EntityManager,
+    job: ProgressJob,
+    ctx: ProgressServiceContext,
+    options: { staleSweptOnly?: boolean } = {},
+  ): Promise<ProgressJob | null> {
+    const now = new Date()
+    // Progress counts are absolute and survive across deliveries, so the elapsed window
+    // calculateEta divides them by has to survive too. Resetting it on every revive makes
+    // a job with hours of work left report seconds remaining.
+    const startedAt = job.startedAt ?? now
+    const filter: Record<string, unknown> = {
+      ...jobScopeFilter(job.id, ctx),
+      status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },
+    }
+    if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` }
+    const affected = await targetEm.nativeUpdate(ProgressJob, filter as FilterQuery<ProgressJob>, {
+      status: 'running',
+      startedAt,
+      heartbeatAt: now,
+      finishedAt: null,
+      errorMessage: null,
+      errorStack: null,
+      updatedAt: now,
+    })
+    if (affected === 0) return null
+
+    job.status = 'running'
+    job.startedAt = startedAt
+    job.heartbeatAt = now
+    job.finishedAt = null
+    job.errorMessage = null
+    job.errorStack = null
+
+    await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
+      ...buildJobPayload(job),
+      tenantId: ctx.tenantId,
+      organizationId: job.organizationId ?? null,
+    })
+
+    return job
+  }
+
   async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {
     const job = entry.job
     const now = Date.now()
@@ -153,16 +206,32 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
       updatedAt: new Date(),
       ...buildBufferedCountData(entry),
     }
-    const affected = await em.nativeUpdate(ProgressJob, {
+    const writableFilter = {
       ...jobScopeFilter(job.id, ctx),
       status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },
-    } as FilterQuery<ProgressJob>, data)
+    } as FilterQuery<ProgressJob>
+    let affected = await em.nativeUpdate(ProgressJob, writableFilter, data)
 
     if (affected === 0) {
-      // The job reached a terminal state in another process; stop writing to it.
-      forgetJobThrottle(job.id)
+      // A stale sweep may have flipped a live job to `failed` between writes. Revive it
+      // through the start CAS and retry once so the buffered delta isn't silently dropped.
       const fresh = await loadFreshJob(job.id, ctx)
-      return fresh ?? job
+      let revived = false
+      if (fresh && fresh.status === 'failed' && (await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true }))) {
+        revived = true
+        job.status = 'running'
+        job.startedAt = fresh.startedAt
+        job.finishedAt = null
+        job.errorMessage = null
+        job.errorStack = null
+        affected = await em.nativeUpdate(ProgressJob, writableFilter, data)
+      }
+      if (affected === 0) {
+        // The job reached a genuinely terminal state in another process; stop writing to it.
+        forgetJobThrottle(job.id)
+        const latest = revived ? await loadFreshJob(job.id, ctx) : fresh
+        return latest ?? job
+      }
     }
 
     const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job
@@ -219,47 +288,48 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
         return job
       }
 
+      const started = await startJobViaCas(em, job, ctx)
+      if (started) return started
+
+      const fresh = await loadFreshJob(jobId, ctx)
+      return fresh ?? job
+    },
+
+    async touchJobHeartbeat(jobId, ctx) {
+      // Forked EM: keepalive timers call this while the shared EM may be mid-transaction
+      // inside a producer's own writes, so the heartbeat must not join that unit of work.
+      const fork = em.fork()
       const now = new Date()
-      const affected = await em.nativeUpdate(ProgressJob, {
+      const affected = await fork.nativeUpdate(ProgressJob, {
         ...jobScopeFilter(jobId, ctx),
-        status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },
+        status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },
       } as FilterQuery<ProgressJob>, {
-        status: 'running',
-        startedAt: now,
         heartbeatAt: now,
-        finishedAt: null,
-        errorMessage: null,
-        errorStack: null,
         updatedAt: now,
       })
+      if (affected > 0) return
 
-      if (affected === 0) {
-        const fresh = await loadFreshJob(jobId, ctx)
-        return fresh ?? job
-      }
-
-      job.status = 'running'
-      job.startedAt = now
-      job.heartbeatAt = now
-      job.finishedAt = null
-      job.errorMessage = null
-      job.errorStack = null
-
-      await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
-        ...buildJobPayload(job),
-        tenantId: ctx.tenantId,
-        organizationId: job.organizationId ?? null,
-      })
-
-      return job
+      const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })
+      if (!job || job.status !== 'failed') return
+      // A live producer heartbeating a `failed` job means a stale sweep false-positived;
+      // revive through the start CAS so the card recovers without waiting for a batch boundary.
+      await startJobViaCas(fork, job, ctx, { staleSweptOnly: true })
     },
 
     async updateProgress(jobId, input, ctx) {
       const entry = await ensureThrottleEntry(jobId, ctx)
       const job = entry.job
       if (TERMINAL_STATUSES.includes(job.status)) {
-        forgetJobThrottle(jobId)
-        return job
+        // `failed` may be a false-positive stale sweep on a slow-but-alive producer; the
+        // fact that this write is happening proves the producer is alive, so revive and
+        // apply the update. `completed`/`cancelled` stay terminal, and a failure the sweep
+        // did not write keeps its diagnostics (see startJobViaCas).
+        const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))
+        if (!revived) {
+          forgetJobThrottle(jobId)
+          return job
+        }
+        entry.lastPersistedAt = 0
       }
 
       if (input.processedCount !== undefined) {
@@ -295,8 +365,12 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
       const entry = await ensureThrottleEntry(jobId, ctx)
       const job = entry.job
       if (TERMINAL_STATUSES.includes(job.status)) {
-        forgetJobThrottle(jobId)
-        return job
+        const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))
+        if (!revived) {
+          forgetJobThrottle(jobId)
+          return job
+        }
+        entry.lastPersistedAt = 0
       }
 
       job.processedCount += delta
@@ -604,7 +678,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
       const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })
 
       for (const job of staleRunning) {
-        const errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`
+        const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`
         // Per-row CAS (re-checking the staleness condition): exactly one concurrent
         // sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.
         const affected = await em.nativeUpdate(ProgressJob, {
@@ -644,7 +718,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
       const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })
 
       for (const job of stalePending) {
-        const errorMessage = `Job stale: never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`
+        const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`
         const affected = await em.nativeUpdate(ProgressJob, {
           id: job.id,
           tenantId: job.tenantId,

package/src/modules/sales/api/channels/route.ts

@@ -38,7 +38,7 @@ const listSchema = z
   .passthrough()
 
 const routeMetadata = {
-  GET: { requireAuth: true, requireFeatures: ['sales.channels.manage'] },
+  GET: { requireAuth: true, requireFeatures: ['sales.channels.view'] },
   POST: { requireAuth: true, requireFeatures: ['sales.channels.manage'] },
   PUT: { requireAuth: true, requireFeatures: ['sales.channels.manage'] },
   DELETE: { requireAuth: true, requireFeatures: ['sales.channels.manage'] },