@open-mercato/core 0.6.6-develop.6416.1.888fffc007 → 0.6.6-develop.6422.1.b3d024f05c

package/src/modules/progress/AGENTS.md

@@ -82,6 +82,12 @@ Use stable, grep-friendly ids:
 | DataTable progress result handling | `packages/ui/src/backend/DataTable.tsx` |
 | Progress UI merge/polling | `packages/ui/src/backend/progress/useProgressPoll.ts` and `useProgressSse.ts` |
 
+## Environment
+
+| Variable | Effect | Default |
+|----------|--------|---------|
+| `OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS` | Coalesces intermediate `progress.job.updated` flush+broadcasts per job: the service flushes and emits only when this many ms elapsed since the last broadcast **or** `progressPercent` advanced by ≥1; sub-threshold updates buffer in memory. Keeps bulk workers from firing one serialized `pg_notify` roundtrip + tenant-wide SSE fan-out per record. Terminal events (`created`/`started`/`completed`/`failed`/`cancelled`) are never throttled, so `ProgressTopBar` still converges and the 60s `STALE_JOB_TIMEOUT_SECONDS` heartbeat window is never exceeded. Set to `0` to restore per-record emission (tests/debugging). | `250` |
+
 ## Cross-References
 
 - **Queue worker rules**: `packages/queue/AGENTS.md`

package/src/modules/progress/lib/progressServiceImpl.ts

@@ -1,10 +1,30 @@
 import type { EntityManager } from '@mikro-orm/postgresql'
 import { ProgressJob } from '../data/entities'
-import type { ProgressService } from './progressService'
+import type { ProgressService, ProgressServiceContext } from './progressService'
 import { calculateEta, calculateProgressPercent, STALE_JOB_TIMEOUT_SECONDS } from './progressService'
 import { PROGRESS_EVENTS } from './events'
 import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
 
+const DEFAULT_BROADCAST_MIN_INTERVAL_MS = 250
+
+// Minimum elapsed time between coalesced `progress.job.updated` flush+broadcasts for a
+// single job. Bulk workers call updateProgress/incrementProgress once per record, and
+// every emit of the `clientBroadcast: true` event pays a serialized pg_notify roundtrip
+// plus a tenant-wide SSE fan-out. Setting the knob to 0 restores per-record emission.
+function resolveBroadcastMinIntervalMs(): number {
+  const raw = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS
+  if (raw == null || raw.trim() === '') return DEFAULT_BROADCAST_MIN_INTERVAL_MS
+  const parsed = Number.parseInt(raw, 10)
+  if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_BROADCAST_MIN_INTERVAL_MS
+  return parsed
+}
+
+type JobUpdateThrottleEntry = {
+  job: ProgressJob
+  lastBroadcastAt: number
+  lastBroadcastPercent: number
+}
+
 function buildJobPayload(job: ProgressJob): Record<string, unknown> {
   return {
     jobId: job.id,
@@ -24,6 +44,60 @@ function buildJobPayload(job: ProgressJob): Record<string, unknown> {
 }
 
 export function createProgressService(em: EntityManager, eventBus: { emit: (event: string, payload: Record<string, unknown>) => Promise<void> }): ProgressService {
+  const broadcastMinIntervalMs = resolveBroadcastMinIntervalMs()
+  // Per-job coalescing state, scoped to this service instance (request/worker scope).
+  // The cached managed entity doubles as the in-memory buffer: intermediate updates mutate
+  // it without flushing, and a single flush+emit at the throttle boundary persists the
+  // accumulated change. Domain commands fork their own EntityManager, so no other operation
+  // queries this job on `em` between calls, keeping the buffered changes safe until flush.
+  const jobUpdateThrottle = new Map<string, JobUpdateThrottleEntry>()
+
+  function jobScopeFilter(jobId: string, ctx: ProgressServiceContext) {
+    return {
+      id: jobId,
+      tenantId: ctx.tenantId,
+      ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),
+    }
+  }
+
+  async function loadUpdatableJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob> {
+    const cached = jobUpdateThrottle.get(jobId)
+    if (cached) return cached.job
+    return em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx))
+  }
+
+  async function commitJobUpdate(job: ProgressJob, ctx: ProgressServiceContext): Promise<ProgressJob> {
+    const now = Date.now()
+    const cached = jobUpdateThrottle.get(job.id)
+    const shouldBroadcast =
+      broadcastMinIntervalMs <= 0 ||
+      cached == null ||
+      now - cached.lastBroadcastAt >= broadcastMinIntervalMs ||
+      Math.abs(job.progressPercent - cached.lastBroadcastPercent) >= 1
+
+    if (shouldBroadcast) {
+      await em.flush()
+      await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {
+        ...buildJobPayload(job),
+        tenantId: ctx.tenantId,
+        organizationId: job.organizationId ?? null,
+      })
+      jobUpdateThrottle.set(job.id, { job, lastBroadcastAt: now, lastBroadcastPercent: job.progressPercent })
+    } else {
+      jobUpdateThrottle.set(job.id, {
+        job,
+        lastBroadcastAt: cached.lastBroadcastAt,
+        lastBroadcastPercent: cached.lastBroadcastPercent,
+      })
+    }
+
+    return job
+  }
+
+  function forgetJobThrottle(jobId: string) {
+    jobUpdateThrottle.delete(jobId)
+  }
+
   return {
     async createJob(input, ctx) {
       const job = em.create(ProgressJob, {
@@ -79,12 +153,9 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
     },
 
     async updateProgress(jobId, input, ctx) {
-      const job = await em.findOneOrFail(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),
-      })
+      const job = await loadUpdatableJob(jobId, ctx)
       if (job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') {
+        forgetJobThrottle(jobId)
         return job
       }
 
@@ -112,24 +183,13 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
 
       job.heartbeatAt = new Date()
 
-      await em.flush()
-
-      await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {
-        ...buildJobPayload(job),
-        tenantId: ctx.tenantId,
-        organizationId: job.organizationId ?? null,
-      })
-
-      return job
+      return commitJobUpdate(job, ctx)
     },
 
     async incrementProgress(jobId, delta, ctx) {
-      const job = await em.findOneOrFail(ProgressJob, {
-        id: jobId,
-        tenantId: ctx.tenantId,
-        ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),
-      })
+      const job = await loadUpdatableJob(jobId, ctx)
       if (job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') {
+        forgetJobThrottle(jobId)
         return job
       }
 
@@ -143,15 +203,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
         }
       }
 
-      await em.flush()
-
-      await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {
-        ...buildJobPayload(job),
-        tenantId: ctx.tenantId,
-        organizationId: job.organizationId ?? null,
-      })
-
-      return job
+      return commitJobUpdate(job, ctx)
     },
 
     async completeJob(jobId, input, ctx) {
@@ -182,6 +234,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
         organizationId: job.organizationId ?? null,
       })
 
+      forgetJobThrottle(jobId)
       return job
     },
 
@@ -210,6 +263,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
         organizationId: job.organizationId ?? null,
       })
 
+      forgetJobThrottle(jobId)
       return job
     },
 
@@ -238,6 +292,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
         organizationId: job.organizationId ?? null,
       })
 
+      forgetJobThrottle(jobId)
       return job
     },
 
@@ -266,6 +321,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
         organizationId: job.organizationId ?? null,
       })
 
+      forgetJobThrottle(jobId)
       return job
     },
 

package/src/modules/staff/api/timesheets/time-entries/route.ts

@@ -4,6 +4,7 @@ import { resolveTranslations } from '@open-mercato/shared/lib/i18n/server'
 import { resolveCrudRecordId, parseScopedCommandInput } from '@open-mercato/shared/lib/api/scoped'
 import { StaffTimeEntry } from '../../../data/entities'
 import { staffTimeEntryCreateSchema, staffTimeEntryUpdateSchema } from '../../../data/validators'
+import { buildTimeEntryListFilters, isParseableDateFilter } from '../../../lib/timesheets/timeEntryListFilters'
 import { createStaffCrudOpenApi, createPagedListResponseSchema, defaultOkResponseSchema } from '../../openapi'
 
 const F = {
@@ -37,13 +38,6 @@ export const metadata = routeMetadata
 
 const rawBodySchema = z.object({}).passthrough()
 
-const isParseableDateFilter = (value: string): boolean => {
-  const trimmed = value.trim()
-  if (trimmed.length === 0) return true
-  if (!/^\d{4}-\d{2}-\d{2}([T ].*)?$/.test(trimmed)) return false
-  return !Number.isNaN(new Date(trimmed).getTime())
-}
-
 const dateFilterSchema = z
   .string()
   .refine(isParseableDateFilter, { message: 'Invalid date' })
@@ -58,6 +52,7 @@ const listSchema = z
     to: dateFilterSchema,
     projectId: z.string().uuid().optional(),
     ids: z.string().optional(),
+    running: z.string().optional(),
     sortField: z.string().optional(),
     sortDir: z.enum(['asc', 'desc']).optional(),
   })
@@ -100,31 +95,7 @@ const crud = makeCrudRoute({
       updatedAt: F.updated_at,
       durationMinutes: F.duration_minutes,
     },
-    buildFilters: async (query) => {
-      const filters: Record<string, unknown> = {}
-      if (typeof query.ids === 'string' && query.ids.trim().length > 0) {
-        const ids = query.ids
-          .split(',')
-          .map((value) => value.trim())
-          .filter((value) => value.length > 0)
-        if (ids.length > 0) {
-          filters[F.id] = { $in: ids }
-        }
-      }
-      if (typeof query.staffMemberId === 'string' && query.staffMemberId.length > 0) {
-        filters[F.staff_member_id] = query.staffMemberId
-      }
-      if (typeof query.from === 'string' && query.from.length > 0 && isParseableDateFilter(query.from)) {
-        filters[F.date] = { ...((filters[F.date] as Record<string, unknown>) ?? {}), $gte: query.from }
-      }
-      if (typeof query.to === 'string' && query.to.length > 0 && isParseableDateFilter(query.to)) {
-        filters[F.date] = { ...((filters[F.date] as Record<string, unknown>) ?? {}), $lte: query.to }
-      }
-      if (typeof query.projectId === 'string' && query.projectId.length > 0) {
-        filters[F.time_project_id] = query.projectId
-      }
-      return filters
-    },
+    buildFilters: async (query) => buildTimeEntryListFilters(query),
   },
   actions: {
     create: {

package/src/modules/staff/lib/timesheets/timeEntryListFilters.ts

@@ -0,0 +1,64 @@
+import { parseBooleanToken } from '@open-mercato/shared/lib/boolean'
+
+export type TimeEntryListFilterQuery = {
+  ids?: string
+  staffMemberId?: string
+  from?: string
+  to?: string
+  projectId?: string
+  running?: string
+}
+
+const ID_COLUMN = 'id'
+const STAFF_MEMBER_COLUMN = 'staff_member_id'
+const DATE_COLUMN = 'date'
+const STARTED_AT_COLUMN = 'started_at'
+const ENDED_AT_COLUMN = 'ended_at'
+const PROJECT_COLUMN = 'time_project_id'
+
+export function isParseableDateFilter(value: string): boolean {
+  const trimmed = value.trim()
+  if (trimmed.length === 0) return true
+  if (!/^\d{4}-\d{2}-\d{2}([T ].*)?$/.test(trimmed)) return false
+  return !Number.isNaN(new Date(trimmed).getTime())
+}
+
+/**
+ * Builds the query-engine filter map for the time-entries list route.
+ *
+ * The `running` flag matches the currently-open timer regardless of its date
+ * (`started_at IS NOT NULL AND ended_at IS NULL`). A timer started before
+ * midnight is still running the next day, so a `from=today&to=today` lookup
+ * would miss it and leave the entry invisible/uncontrollable (issue #3717).
+ * This mirrors the single-active-timer check in the `start_timer` command so
+ * the UI can always surface and stop an orphaned overnight timer.
+ */
+export function buildTimeEntryListFilters(query: TimeEntryListFilterQuery): Record<string, unknown> {
+  const filters: Record<string, unknown> = {}
+  if (typeof query.ids === 'string' && query.ids.trim().length > 0) {
+    const ids = query.ids
+      .split(',')
+      .map((value) => value.trim())
+      .filter((value) => value.length > 0)
+    if (ids.length > 0) {
+      filters[ID_COLUMN] = { $in: ids }
+    }
+  }
+  if (typeof query.staffMemberId === 'string' && query.staffMemberId.length > 0) {
+    filters[STAFF_MEMBER_COLUMN] = query.staffMemberId
+  }
+  if (typeof query.from === 'string' && query.from.length > 0 && isParseableDateFilter(query.from)) {
+    filters[DATE_COLUMN] = { ...((filters[DATE_COLUMN] as Record<string, unknown>) ?? {}), $gte: query.from }
+  }
+  if (typeof query.to === 'string' && query.to.length > 0 && isParseableDateFilter(query.to)) {
+    filters[DATE_COLUMN] = { ...((filters[DATE_COLUMN] as Record<string, unknown>) ?? {}), $lte: query.to }
+  }
+  if (typeof query.projectId === 'string' && query.projectId.length > 0) {
+    filters[PROJECT_COLUMN] = query.projectId
+  }
+  if (parseBooleanToken(query.running ?? null) === true) {
+    filters[STARTED_AT_COLUMN] = { $ne: null }
+    filters[ENDED_AT_COLUMN] = null
+  }
+  return filters
+}

package/src/modules/staff/lib/timesheets-ui/TimerBar.tsx

@@ -102,10 +102,9 @@ export function TimerBar({ projects, staffMemberId, onTimerStopped }: TimerBarPr
   useEffect(() => {
     if (!staffMemberId) return
 
-    const today = getToday()
     const checkActiveTimer = async () => {
       const response = await apiCall(
-        `/api/staff/timesheets/time-entries?staffMemberId=${staffMemberId}&from=${today}&to=${today}&pageSize=50`,
+        `/api/staff/timesheets/time-entries?staffMemberId=${staffMemberId}&running=true&pageSize=50`,
       )
       if (!response.ok) return
 

package/src/modules/staff/widgets/dashboard/timesheets-time-reporting/widget.client.tsx

@@ -91,9 +91,7 @@ const TimeReportingWidget: React.FC<DashboardWidgetComponentProps<TimeReportingS
       const memberId = selfRes.member?.id ?? null
       setStaffMemberId(memberId)
 
-      const today = new Date().toISOString().slice(0, 10)
-
-      // Project details depend on the assignment ids and active entries depend on the staff
+      // Project details depend on the assignment ids and the active timer depends on the staff
       // member id, but the two requests are independent of each other — fetch them together.
       const [projectsRes, entriesRes] = await Promise.all([
         projectIds.length > 0
@@ -105,7 +103,7 @@ const TimeReportingWidget: React.FC<DashboardWidgetComponentProps<TimeReportingS
           : Promise.resolve<{ items?: Array<Record<string, unknown>> }>({ items: [] }),
         memberId
           ? readApiResultOrThrow<{ items?: Array<Record<string, unknown>> }>(
-              `/api/staff/timesheets/time-entries?staffMemberId=${memberId}&from=${today}&to=${today}&pageSize=100`,
+              `/api/staff/timesheets/time-entries?staffMemberId=${memberId}&running=true&pageSize=100`,
               undefined,
               { errorMessage: '', fallback: { items: [] } },
             )

package/src/modules/staff/widgets/injection/timer-sidebar-indicator/widget.tsx

@@ -39,11 +39,6 @@ function formatElapsed(seconds: number): string {
   return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`
 }
 
-function getToday(): string {
-  const now = new Date()
-  return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`
-}
-
 function TimerSidebarIndicator() {
   const t = useT()
   // Initialise from sessionStorage so the indicator doesn't flash away on navigation
@@ -63,9 +58,8 @@ function TimerSidebarIndicator() {
       const memberId = selfRes.result?.member?.id
       if (!memberId) { updateTimer(null); return }
 
-      const today = getToday()
       const res = await apiCall<{ items?: Array<Record<string, unknown>> }>(
-        `/api/staff/timesheets/time-entries?staffMemberId=${memberId}&from=${today}&to=${today}&pageSize=50`,
+        `/api/staff/timesheets/time-entries?staffMemberId=${memberId}&running=true&pageSize=50`,
       )
       const items = (res.result?.items ?? []) as Array<Record<string, unknown>>
       const active = items.find((e) => e.started_at && !e.ended_at)

package/src/modules/sync_excel/lib/adapters/customers.ts

@@ -1048,15 +1048,15 @@ export const syncExcelCustomersAdapter: DataSyncAdapter = {
       const startOffset = cursor?.uploadId === upload.id ? cursor.offset : 0
       const commandContext = buildCommandContext(container, input.scope)
       const customFieldDefinitions = await loadImportCustomFieldDefinitions(em, input.scope)
+      const emailDedupeIndex = await buildEmailDedupeIndex({
+        em,
+        rows: document.rows.slice(startOffset),
+        mapping: input.mapping,
+        scope: input.scope,
+      })
 
       for (let offset = startOffset, batchIndex = 0; offset < document.rows.length; offset += input.batchSize, batchIndex += 1) {
         const batchRows = document.rows.slice(offset, offset + input.batchSize)
-        const emailDedupeIndex = await buildEmailDedupeIndex({
-          em,
-          rows: batchRows,
-          mapping: input.mapping,
-          scope: input.scope,
-        })
         const items: ImportItem[] = []
 
         for (let index = 0; index < batchRows.length; index += 1) {