@open-mercato/core 0.6.7-develop.6726.1.983ae8a07e → 0.6.7-develop.6749.1.6b54c56dfe

package/dist/modules/progress/lib/progressServiceImpl.js

@@ -1,15 +1,28 @@
+import { raw } from "@mikro-orm/core";
 import { ProgressJob } from "../data/entities.js";
-import { calculateEta, calculateProgressPercent, STALE_JOB_TIMEOUT_SECONDS } from "./progressService.js";
+import {
+  calculateEta,
+  calculateProgressPercent,
+  HEARTBEAT_INTERVAL_MS,
+  STALE_JOB_TIMEOUT_SECONDS,
+  STALE_PENDING_TIMEOUT_SECONDS
+} from "./progressService.js";
 import { PROGRESS_EVENTS } from "./events.js";
 import { findOneWithDecryption } from "@open-mercato/shared/lib/encryption/find";
 const DEFAULT_BROADCAST_MIN_INTERVAL_MS = 250;
 function resolveBroadcastMinIntervalMs() {
-  const raw = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS;
-  if (raw == null || raw.trim() === "") return DEFAULT_BROADCAST_MIN_INTERVAL_MS;
-  const parsed = Number.parseInt(raw, 10);
+  const rawValue = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS;
+  if (rawValue == null || rawValue.trim() === "") return DEFAULT_BROADCAST_MIN_INTERVAL_MS;
+  const parsed = Number.parseInt(rawValue, 10);
   if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_BROADCAST_MIN_INTERVAL_MS;
   return parsed;
 }
+const TERMINAL_STATUSES = ["completed", "failed", "cancelled"];
+const PROGRESS_WRITABLE_STATUSES = ["pending", "running"];
+const START_FROM_STATUSES = ["pending", "failed"];
+const COMPLETE_FROM_STATUSES = ["pending", "running", "failed"];
+const FAIL_FROM_STATUSES = ["pending", "running"];
+const CANCEL_FROM_STATUSES = ["pending", "running", "failed"];
 function buildJobPayload(job) {
   return {
     jobId: job.id,
@@ -37,34 +50,86 @@ function createProgressService(em, eventBus) {
       ...ctx.organizationId ? { organizationId: ctx.organizationId } : {}
     };
   }
-  async function loadUpdatableJob(jobId, ctx) {
+  async function loadFreshJob(jobId, ctx) {
+    return em.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true });
+  }
+  async function ensureThrottleEntry(jobId, ctx) {
     const cached = jobUpdateThrottle.get(jobId);
-    if (cached) return cached.job;
-    return em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx));
+    if (cached) return cached;
+    const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true });
+    const entry = {
+      job,
+      lastBroadcastAt: 0,
+      lastPersistedAt: 0,
+      lastBroadcastPercent: job.progressPercent,
+      pendingDelta: 0,
+      absoluteCountsPending: false
+    };
+    jobUpdateThrottle.set(jobId, entry);
+    return entry;
+  }
+  function forgetJobThrottle(jobId) {
+    jobUpdateThrottle.delete(jobId);
   }
-  async function commitJobUpdate(job, ctx) {
+  function buildBufferedCountData(entry) {
+    if (entry.absoluteCountsPending || !Number.isSafeInteger(entry.pendingDelta)) {
+      return { processedCount: entry.job.processedCount };
+    }
+    if (entry.pendingDelta !== 0) {
+      const processedCount = raw(`processed_count + ${entry.pendingDelta}`);
+      const totalCount = entry.job.totalCount;
+      if (Number.isSafeInteger(totalCount) && totalCount != null && totalCount > 0) {
+        return {
+          processedCount,
+          progressPercent: raw(
+            `least(100, round(((processed_count + ${entry.pendingDelta})::numeric / ${totalCount}) * 100))`
+          )
+        };
+      }
+      return { processedCount };
+    }
+    return {};
+  }
+  async function persistAndMaybeBroadcast(entry, ctx) {
+    const job = entry.job;
     const now = Date.now();
-    const cached = jobUpdateThrottle.get(job.id);
-    const shouldBroadcast = broadcastMinIntervalMs <= 0 || cached == null || now - cached.lastBroadcastAt >= broadcastMinIntervalMs || Math.abs(job.progressPercent - cached.lastBroadcastPercent) >= 1;
+    const shouldBroadcast = broadcastMinIntervalMs <= 0 || now - entry.lastBroadcastAt >= broadcastMinIntervalMs || Math.abs(job.progressPercent - entry.lastBroadcastPercent) >= 1;
+    const shouldPersist = shouldBroadcast || now - entry.lastPersistedAt >= HEARTBEAT_INTERVAL_MS;
+    if (!shouldPersist) return job;
+    const usesAtomicIncrement = !entry.absoluteCountsPending && Number.isSafeInteger(entry.pendingDelta) && entry.pendingDelta !== 0;
+    const data = {
+      progressPercent: job.progressPercent,
+      totalCount: job.totalCount ?? null,
+      etaSeconds: job.etaSeconds ?? null,
+      meta: job.meta ?? null,
+      heartbeatAt: job.heartbeatAt ?? /* @__PURE__ */ new Date(),
+      updatedAt: /* @__PURE__ */ new Date(),
+      ...buildBufferedCountData(entry)
+    };
+    const affected = await em.nativeUpdate(ProgressJob, {
+      ...jobScopeFilter(job.id, ctx),
+      status: { $in: PROGRESS_WRITABLE_STATUSES }
+    }, data);
+    if (affected === 0) {
+      forgetJobThrottle(job.id);
+      const fresh = await loadFreshJob(job.id, ctx);
+      return fresh ?? job;
+    }
+    const persistedJob = usesAtomicIncrement ? await loadFreshJob(job.id, ctx) ?? job : job;
+    entry.job = persistedJob;
+    entry.pendingDelta = 0;
+    entry.absoluteCountsPending = false;
+    entry.lastPersistedAt = now;
     if (shouldBroadcast) {
-      await em.flush();
       await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {
-        ...buildJobPayload(job),
+        ...buildJobPayload(persistedJob),
         tenantId: ctx.tenantId,
-        organizationId: job.organizationId ?? null
-      });
-      jobUpdateThrottle.set(job.id, { job, lastBroadcastAt: now, lastBroadcastPercent: job.progressPercent });
-    } else {
-      jobUpdateThrottle.set(job.id, {
-        job,
-        lastBroadcastAt: cached.lastBroadcastAt,
-        lastBroadcastPercent: cached.lastBroadcastPercent
+        organizationId: persistedJob.organizationId ?? null
       });
+      entry.lastBroadcastAt = now;
+      entry.lastBroadcastPercent = persistedJob.progressPercent;
     }
-    return job;
-  }
-  function forgetJobThrottle(jobId) {
-    jobUpdateThrottle.delete(jobId);
+    return persistedJob;
   }
   return {
     async createJob(input, ctx) {
@@ -92,18 +157,33 @@ function createProgressService(em, eventBus) {
       return job;
     },
     async startJob(jobId, ctx) {
-      const job = await em.findOneOrFail(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...ctx.organizationId ? { organizationId: ctx.organizationId } : {}
-      });
-      if (job.status === "cancelled") {
+      const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true });
+      if (job.status === "running" || job.status === "cancelled" || job.status === "completed") {
         return job;
       }
+      const now = /* @__PURE__ */ new Date();
+      const affected = await em.nativeUpdate(ProgressJob, {
+        ...jobScopeFilter(jobId, ctx),
+        status: { $in: START_FROM_STATUSES }
+      }, {
+        status: "running",
+        startedAt: now,
+        heartbeatAt: now,
+        finishedAt: null,
+        errorMessage: null,
+        errorStack: null,
+        updatedAt: now
+      });
+      if (affected === 0) {
+        const fresh = await loadFreshJob(jobId, ctx);
+        return fresh ?? job;
+      }
       job.status = "running";
-      job.startedAt = /* @__PURE__ */ new Date();
-      job.heartbeatAt = /* @__PURE__ */ new Date();
-      await em.flush();
+      job.startedAt = now;
+      job.heartbeatAt = now;
+      job.finishedAt = null;
+      job.errorMessage = null;
+      job.errorStack = null;
       await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
         ...buildJobPayload(job),
         tenantId: ctx.tenantId,
@@ -112,13 +192,16 @@ function createProgressService(em, eventBus) {
       return job;
     },
     async updateProgress(jobId, input, ctx) {
-      const job = await loadUpdatableJob(jobId, ctx);
-      if (job.status === "completed" || job.status === "failed" || job.status === "cancelled") {
+      const entry = await ensureThrottleEntry(jobId, ctx);
+      const job = entry.job;
+      if (TERMINAL_STATUSES.includes(job.status)) {
         forgetJobThrottle(jobId);
         return job;
       }
       if (input.processedCount !== void 0) {
         job.processedCount = input.processedCount;
+        entry.absoluteCountsPending = true;
+        entry.pendingDelta = 0;
       }
       if (input.totalCount !== void 0) {
         job.totalCount = input.totalCount;
@@ -137,15 +220,17 @@ function createProgressService(em, eventBus) {
         job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt);
       }
       job.heartbeatAt = /* @__PURE__ */ new Date();
-      return commitJobUpdate(job, ctx);
+      return persistAndMaybeBroadcast(entry, ctx);
     },
     async incrementProgress(jobId, delta, ctx) {
-      const job = await loadUpdatableJob(jobId, ctx);
-      if (job.status === "completed" || job.status === "failed" || job.status === "cancelled") {
+      const entry = await ensureThrottleEntry(jobId, ctx);
+      const job = entry.job;
+      if (TERMINAL_STATUSES.includes(job.status)) {
         forgetJobThrottle(jobId);
         return job;
       }
       job.processedCount += delta;
+      entry.pendingDelta += delta;
       job.heartbeatAt = /* @__PURE__ */ new Date();
       if (job.totalCount) {
         job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount);
@@ -153,104 +238,189 @@ function createProgressService(em, eventBus) {
           job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt);
         }
       }
-      return commitJobUpdate(job, ctx);
+      return persistAndMaybeBroadcast(entry, ctx);
     },
     async completeJob(jobId, input, ctx) {
-      const job = await em.findOne(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...ctx.organizationId ? { organizationId: ctx.organizationId } : {}
-      });
+      const job = await loadFreshJob(jobId, ctx);
       if (!job) throw new Error(`Job ${jobId} not found`);
-      if (job.status === "cancelled") {
+      if (job.status === "cancelled" || job.status === "completed") {
+        forgetJobThrottle(jobId);
         return job;
       }
-      job.status = "completed";
-      job.finishedAt = /* @__PURE__ */ new Date();
-      job.progressPercent = 100;
-      job.etaSeconds = 0;
+      const entry = jobUpdateThrottle.get(jobId);
+      const snapshot = entry?.job ?? job;
+      const now = /* @__PURE__ */ new Date();
+      const data = {
+        status: "completed",
+        finishedAt: now,
+        etaSeconds: 0,
+        updatedAt: now,
+        ...input?.resultSummary ? { resultSummary: input.resultSummary } : {},
+        ...entry ? {
+          totalCount: snapshot.totalCount ?? null,
+          meta: snapshot.meta ?? null,
+          ...buildBufferedCountData(entry)
+        } : {},
+        progressPercent: 100
+      };
+      const affected = await em.nativeUpdate(ProgressJob, {
+        ...jobScopeFilter(jobId, ctx),
+        status: { $in: COMPLETE_FROM_STATUSES }
+      }, data);
+      forgetJobThrottle(jobId);
+      if (affected === 0) {
+        const fresh = await loadFreshJob(jobId, ctx);
+        return fresh ?? job;
+      }
+      snapshot.status = "completed";
+      snapshot.finishedAt = now;
+      snapshot.progressPercent = 100;
+      snapshot.etaSeconds = 0;
       if (input?.resultSummary) {
-        job.resultSummary = input.resultSummary;
+        snapshot.resultSummary = input.resultSummary;
       }
-      await em.flush();
+      const persistedJob = await loadFreshJob(jobId, ctx) ?? snapshot;
       await eventBus.emit(PROGRESS_EVENTS.JOB_COMPLETED, {
-        ...buildJobPayload(job),
-        resultSummary: job.resultSummary,
+        ...buildJobPayload(persistedJob),
+        resultSummary: persistedJob.resultSummary,
         tenantId: ctx.tenantId,
-        organizationId: job.organizationId ?? null
+        organizationId: persistedJob.organizationId ?? null
       });
-      forgetJobThrottle(jobId);
-      return job;
+      return persistedJob;
     },
     async failJob(jobId, input, ctx) {
-      const job = await em.findOne(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...ctx.organizationId ? { organizationId: ctx.organizationId } : {}
-      });
+      const job = await loadFreshJob(jobId, ctx);
       if (!job) throw new Error(`Job ${jobId} not found`);
-      if (job.status === "cancelled") {
+      if (TERMINAL_STATUSES.includes(job.status)) {
+        forgetJobThrottle(jobId);
         return job;
       }
-      job.status = "failed";
-      job.finishedAt = /* @__PURE__ */ new Date();
-      job.errorMessage = input.errorMessage;
-      job.errorStack = input.errorStack;
-      await em.flush();
+      const entry = jobUpdateThrottle.get(jobId);
+      const snapshot = entry?.job ?? job;
+      const now = /* @__PURE__ */ new Date();
+      const data = {
+        status: "failed",
+        finishedAt: now,
+        errorMessage: input.errorMessage,
+        errorStack: input.errorStack,
+        updatedAt: now,
+        ...entry ? {
+          totalCount: snapshot.totalCount ?? null,
+          meta: snapshot.meta ?? null,
+          ...buildBufferedCountData(entry)
+        } : {}
+      };
+      const affected = await em.nativeUpdate(ProgressJob, {
+        ...jobScopeFilter(jobId, ctx),
+        status: { $in: FAIL_FROM_STATUSES }
+      }, data);
+      forgetJobThrottle(jobId);
+      if (affected === 0) {
+        const fresh = await loadFreshJob(jobId, ctx);
+        return fresh ?? job;
+      }
+      snapshot.status = "failed";
+      snapshot.finishedAt = now;
+      snapshot.errorMessage = input.errorMessage;
+      snapshot.errorStack = input.errorStack;
+      const persistedJob = await loadFreshJob(jobId, ctx) ?? snapshot;
       await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {
-        ...buildJobPayload(job),
-        errorMessage: job.errorMessage,
+        ...buildJobPayload(persistedJob),
+        errorMessage: persistedJob.errorMessage,
         tenantId: ctx.tenantId,
-        organizationId: job.organizationId ?? null
+        organizationId: persistedJob.organizationId ?? null
       });
-      forgetJobThrottle(jobId);
-      return job;
+      return persistedJob;
     },
     async cancelJob(jobId, ctx) {
-      const job = await em.findOneOrFail(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...ctx.organizationId ? { organizationId: ctx.organizationId } : {},
+      const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true });
+      if (TERMINAL_STATUSES.includes(job.status)) {
+        return job;
+      }
+      if (!job.cancellable) {
+        throw new Error(`Job ${jobId} is not cancellable`);
+      }
+      const now = /* @__PURE__ */ new Date();
+      const cancelledNow = await em.nativeUpdate(ProgressJob, {
+        ...jobScopeFilter(jobId, ctx),
         cancellable: true,
-        status: { $in: ["pending", "running"] }
+        status: "pending"
+      }, {
+        status: "cancelled",
+        cancelRequestedAt: now,
+        cancelledByUserId: ctx.userId ?? null,
+        finishedAt: now,
+        etaSeconds: 0,
+        updatedAt: now
       });
-      job.cancelRequestedAt = /* @__PURE__ */ new Date();
-      job.cancelledByUserId = ctx.userId;
-      if (job.status === "pending") {
+      let requested = 0;
+      if (cancelledNow === 0) {
+        requested = await em.nativeUpdate(ProgressJob, {
+          ...jobScopeFilter(jobId, ctx),
+          cancellable: true,
+          status: "running"
+        }, {
+          cancelRequestedAt: now,
+          cancelledByUserId: ctx.userId ?? null,
+          updatedAt: now
+        });
+      }
+      if (cancelledNow === 0 && requested === 0) {
+        const fresh = await loadFreshJob(jobId, ctx);
+        return fresh ?? job;
+      }
+      forgetJobThrottle(jobId);
+      if (cancelledNow > 0) {
         job.status = "cancelled";
-        job.finishedAt = /* @__PURE__ */ new Date();
+        job.finishedAt = now;
+        job.etaSeconds = 0;
       }
-      await em.flush();
+      job.cancelRequestedAt = now;
+      job.cancelledByUserId = ctx.userId ?? null;
       await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {
         ...buildJobPayload(job),
         tenantId: ctx.tenantId,
         organizationId: job.organizationId ?? null
       });
-      forgetJobThrottle(jobId);
       return job;
     },
     async markCancelled(jobId, ctx) {
-      const job = await em.findOne(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...ctx.organizationId ? { organizationId: ctx.organizationId } : {}
-      });
+      const job = await loadFreshJob(jobId, ctx);
       if (!job) throw new Error(`Job ${jobId} not found`);
-      if (job.status === "cancelled") {
+      if (job.status === "cancelled" || job.status === "completed") {
+        forgetJobThrottle(jobId);
         return job;
       }
-      job.cancelRequestedAt = job.cancelRequestedAt ?? /* @__PURE__ */ new Date();
-      job.cancelledByUserId = ctx.userId;
+      const now = /* @__PURE__ */ new Date();
+      const cancelRequestedAt = job.cancelRequestedAt ?? now;
+      const cancelledByUserId = job.cancelledByUserId ?? ctx.userId ?? null;
+      const finishedAt = job.finishedAt ?? now;
+      const affected = await em.nativeUpdate(ProgressJob, {
+        ...jobScopeFilter(jobId, ctx),
+        status: { $in: CANCEL_FROM_STATUSES }
+      }, {
+        status: "cancelled",
+        cancelRequestedAt,
+        cancelledByUserId,
+        finishedAt,
+        etaSeconds: 0,
+        updatedAt: now
+      });
+      forgetJobThrottle(jobId);
+      if (affected === 0) {
+        const fresh = await loadFreshJob(jobId, ctx);
+        return fresh ?? job;
+      }
       job.status = "cancelled";
-      job.finishedAt = job.finishedAt ?? /* @__PURE__ */ new Date();
+      job.cancelRequestedAt = cancelRequestedAt;
+      job.cancelledByUserId = cancelledByUserId;
+      job.finishedAt = finishedAt;
       job.etaSeconds = 0;
-      await em.flush();
       await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {
         ...buildJobPayload(job),
         tenantId: ctx.tenantId,
         organizationId: job.organizationId ?? null
       });
-      forgetJobThrottle(jobId);
       return job;
     },
     async isCancellationRequested(jobId, tenantId, organizationId) {
@@ -258,7 +428,7 @@ function createProgressService(em, eventBus) {
         id: jobId,
         tenantId,
         ...organizationId ? { organizationId } : {}
-      });
+      }, { disableIdentityMap: true });
       return job?.cancelRequestedAt != null;
     },
     async getActiveJobs(ctx) {
@@ -293,10 +463,14 @@ function createProgressService(em, eventBus) {
       });
     },
     async markStaleJobsFailed(tenantId, timeoutSeconds = STALE_JOB_TIMEOUT_SECONDS, organizationId) {
-      const cutoff = new Date(Date.now() - timeoutSeconds * 1e3);
-      const staleJobs = await em.find(ProgressJob, {
+      const now = /* @__PURE__ */ new Date();
+      const cutoff = new Date(now.getTime() - timeoutSeconds * 1e3);
+      const scope = {
         tenantId,
-        ...organizationId ? { organizationId } : {},
+        ...organizationId ? { organizationId } : {}
+      };
+      let failedCount = 0;
+      const staleFilter = {
         status: "running",
         $or: [
           { heartbeatAt: { $lt: cutoff } },
@@ -305,11 +479,56 @@ function createProgressService(em, eventBus) {
             startedAt: { $lt: cutoff }
           }
         ]
-      });
-      for (const job of staleJobs) {
+      };
+      const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true });
+      for (const job of staleRunning) {
+        const errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`;
+        const affected = await em.nativeUpdate(ProgressJob, {
+          id: job.id,
+          tenantId: job.tenantId,
+          ...staleFilter
+        }, {
+          status: "failed",
+          finishedAt: now,
+          errorMessage,
+          updatedAt: now
+        });
+        if (affected === 0) continue;
+        failedCount += 1;
+        job.status = "failed";
+        job.finishedAt = now;
+        job.errorMessage = errorMessage;
+        await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {
+          ...buildJobPayload(job),
+          errorMessage: job.errorMessage,
+          tenantId: job.tenantId,
+          stale: true,
+          organizationId: job.organizationId ?? null
+        });
+      }
+      const pendingCutoff = new Date(now.getTime() - STALE_PENDING_TIMEOUT_SECONDS * 1e3);
+      const stalePendingFilter = {
+        status: "pending",
+        createdAt: { $lt: pendingCutoff }
+      };
+      const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true });
+      for (const job of stalePending) {
+        const errorMessage = `Job stale: never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`;
+        const affected = await em.nativeUpdate(ProgressJob, {
+          id: job.id,
+          tenantId: job.tenantId,
+          ...stalePendingFilter
+        }, {
+          status: "failed",
+          finishedAt: now,
+          errorMessage,
+          updatedAt: now
+        });
+        if (affected === 0) continue;
+        failedCount += 1;
         job.status = "failed";
-        job.finishedAt = /* @__PURE__ */ new Date();
-        job.errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`;
+        job.finishedAt = now;
+        job.errorMessage = errorMessage;
         await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {
           ...buildJobPayload(job),
           errorMessage: job.errorMessage,
@@ -318,8 +537,7 @@ function createProgressService(em, eventBus) {
           organizationId: job.organizationId ?? null
         });
       }
-      await em.flush();
-      return staleJobs.length;
+      return failedCount;
     }
   };
 }