@open-mercato/core 0.6.7-develop.6726.1.983ae8a07e → 0.6.7-develop.6749.1.6b54c56dfe

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/dist/generated/entities/resources_resource/index.js +2 -0
  3. package/dist/generated/entities/resources_resource/index.js.map +2 -2
  4. package/dist/generated/entity-fields-registry.js +1 -0
  5. package/dist/generated/entity-fields-registry.js.map +2 -2
  6. package/dist/helpers/integration/salesFixtures.js +31 -2
  7. package/dist/helpers/integration/salesFixtures.js.map +2 -2
  8. package/dist/helpers/integration/salesUi.js +5 -0
  9. package/dist/helpers/integration/salesUi.js.map +2 -2
  10. package/dist/modules/auth/api/profile/route.js +8 -2
  11. package/dist/modules/auth/api/profile/route.js.map +2 -2
  12. package/dist/modules/auth/backend/auth/profile/page.js.map +2 -2
  13. package/dist/modules/auth/backend/profile/change-password/page.js.map +2 -2
  14. package/dist/modules/communication_channels/data/enrichers.js +4 -4
  15. package/dist/modules/communication_channels/data/enrichers.js.map +2 -2
  16. package/dist/modules/customers/analytics.js +1 -0
  17. package/dist/modules/customers/analytics.js.map +2 -2
  18. package/dist/modules/customers/components/detail/DealForm.js +2 -1
  19. package/dist/modules/customers/components/detail/DealForm.js.map +2 -2
  20. package/dist/modules/customers/components/detail/DealsSection.js +1 -1
  21. package/dist/modules/customers/components/detail/DealsSection.js.map +2 -2
  22. package/dist/modules/customers/data/validators.js +3 -1
  23. package/dist/modules/customers/data/validators.js.map +2 -2
  24. package/dist/modules/dashboards/widgets/dashboard/pipeline-summary/config.js +52 -2
  25. package/dist/modules/dashboards/widgets/dashboard/pipeline-summary/config.js.map +2 -2
  26. package/dist/modules/dashboards/widgets/dashboard/pipeline-summary/widget.client.js +42 -26
  27. package/dist/modules/dashboards/widgets/dashboard/pipeline-summary/widget.client.js.map +2 -2
  28. package/dist/modules/dashboards/widgets/dashboard/pipeline-summary/widget.js +2 -2
  29. package/dist/modules/dashboards/widgets/dashboard/pipeline-summary/widget.js.map +2 -2
  30. package/dist/modules/data_sync/lib/queue-policy.js +15 -0
  31. package/dist/modules/data_sync/lib/queue-policy.js.map +7 -0
  32. package/dist/modules/data_sync/lib/queue.js +16 -1
  33. package/dist/modules/data_sync/lib/queue.js.map +2 -2
  34. package/dist/modules/data_sync/lib/start-run.js +2 -1
  35. package/dist/modules/data_sync/lib/start-run.js.map +2 -2
  36. package/dist/modules/data_sync/lib/sync-engine.js +35 -2
  37. package/dist/modules/data_sync/lib/sync-engine.js.map +2 -2
  38. package/dist/modules/data_sync/lib/sync-run-service.js +70 -2
  39. package/dist/modules/data_sync/lib/sync-run-service.js.map +2 -2
  40. package/dist/modules/data_sync/workers/sync-export.js +9 -2
  41. package/dist/modules/data_sync/workers/sync-export.js.map +2 -2
  42. package/dist/modules/data_sync/workers/sync-import.js +9 -2
  43. package/dist/modules/data_sync/workers/sync-import.js.map +2 -2
  44. package/dist/modules/dictionaries/api/[dictionaryId]/entries/route.js +34 -12
  45. package/dist/modules/dictionaries/api/[dictionaryId]/entries/route.js.map +2 -2
  46. package/dist/modules/dictionaries/api/openapi.js +15 -2
  47. package/dist/modules/dictionaries/api/openapi.js.map +2 -2
  48. package/dist/modules/dictionaries/components/hooks/useDictionaryEntries.js +30 -29
  49. package/dist/modules/dictionaries/components/hooks/useDictionaryEntries.js.map +2 -2
  50. package/dist/modules/dictionaries/data/validators.js +7 -0
  51. package/dist/modules/dictionaries/data/validators.js.map +2 -2
  52. package/dist/modules/dictionaries/lib/clientEntries.js +4 -3
  53. package/dist/modules/dictionaries/lib/clientEntries.js.map +2 -2
  54. package/dist/modules/dictionaries/lib/fetchAllEntries.js +43 -0
  55. package/dist/modules/dictionaries/lib/fetchAllEntries.js.map +7 -0
  56. package/dist/modules/entities/api/definitions.js +51 -3
  57. package/dist/modules/entities/api/definitions.js.map +2 -2
  58. package/dist/modules/entities/api/entities.js +21 -2
  59. package/dist/modules/entities/api/entities.js.map +2 -2
  60. package/dist/modules/entities/api/records.js +3 -24
  61. package/dist/modules/entities/api/records.js.map +2 -2
  62. package/dist/modules/entities/api/sidebar-entities.js +34 -12
  63. package/dist/modules/entities/api/sidebar-entities.js.map +2 -2
  64. package/dist/modules/entities/lib/entityAcl.js +46 -0
  65. package/dist/modules/entities/lib/entityAcl.js.map +2 -2
  66. package/dist/modules/messages/api/route.js +2 -5
  67. package/dist/modules/messages/api/route.js.map +2 -2
  68. package/dist/modules/messages/lib/participantScope.js +15 -0
  69. package/dist/modules/messages/lib/participantScope.js.map +7 -0
  70. package/dist/modules/notifications/api/route.js +3 -1
  71. package/dist/modules/notifications/api/route.js.map +2 -2
  72. package/dist/modules/notifications/api/unread-count/route.js +25 -6
  73. package/dist/modules/notifications/api/unread-count/route.js.map +2 -2
  74. package/dist/modules/notifications/lib/notificationScope.js +26 -0
  75. package/dist/modules/notifications/lib/notificationScope.js.map +7 -0
  76. package/dist/modules/notifications/lib/notificationService.js +7 -3
  77. package/dist/modules/notifications/lib/notificationService.js.map +2 -2
  78. package/dist/modules/notifications/lib/routeHelpers.js +16 -2
  79. package/dist/modules/notifications/lib/routeHelpers.js.map +2 -2
  80. package/dist/modules/planner/components/unavailabilityReasons.js +3 -3
  81. package/dist/modules/planner/components/unavailabilityReasons.js.map +2 -2
  82. package/dist/modules/progress/lib/progressService.js +2 -0
  83. package/dist/modules/progress/lib/progressService.js.map +2 -2
  84. package/dist/modules/progress/lib/progressServiceImpl.js +322 -104
  85. package/dist/modules/progress/lib/progressServiceImpl.js.map +2 -2
  86. package/dist/modules/resources/api/resources.js +2 -0
  87. package/dist/modules/resources/api/resources.js.map +2 -2
  88. package/dist/modules/resources/backend/resources/resources/[id]/page.js +3 -2
  89. package/dist/modules/resources/backend/resources/resources/[id]/page.js.map +2 -2
  90. package/dist/modules/resources/commands/resources.js +9 -0
  91. package/dist/modules/resources/commands/resources.js.map +2 -2
  92. package/dist/modules/resources/components/detail/dictionaries.js +3 -3
  93. package/dist/modules/resources/components/detail/dictionaries.js.map +2 -2
  94. package/dist/modules/resources/data/entities.js +3 -0
  95. package/dist/modules/resources/data/entities.js.map +2 -2
  96. package/dist/modules/resources/data/validators.js +6 -2
  97. package/dist/modules/resources/data/validators.js.map +2 -2
  98. package/dist/modules/resources/lib/seeds.js +16 -9
  99. package/dist/modules/resources/lib/seeds.js.map +2 -2
  100. package/dist/modules/resources/migrations/Migration20260608231000.js +13 -0
  101. package/dist/modules/resources/migrations/Migration20260608231000.js.map +7 -0
  102. package/dist/modules/sales/backend/sales/documents/create/page.js +11 -14
  103. package/dist/modules/sales/backend/sales/documents/create/page.js.map +2 -2
  104. package/dist/modules/sales/commands/documents.js +8 -0
  105. package/dist/modules/sales/commands/documents.js.map +2 -2
  106. package/dist/modules/sales/components/documents/ItemsSection.js +18 -9
  107. package/dist/modules/sales/components/documents/ItemsSection.js.map +2 -2
  108. package/dist/modules/sales/components/documents/LineItemDialog.js +12 -5
  109. package/dist/modules/sales/components/documents/LineItemDialog.js.map +2 -2
  110. package/dist/modules/sales/components/documents/SalesDocumentForm.js +48 -2
  111. package/dist/modules/sales/components/documents/SalesDocumentForm.js.map +2 -2
  112. package/dist/modules/sales/components/documents/SalesOrderDraftLines.js +167 -0
  113. package/dist/modules/sales/components/documents/SalesOrderDraftLines.js.map +7 -0
  114. package/dist/modules/sales/data/validators.js +5 -1
  115. package/dist/modules/sales/data/validators.js.map +2 -2
  116. package/dist/modules/staff/components/detail/dictionaries.js +3 -3
  117. package/dist/modules/staff/components/detail/dictionaries.js.map +2 -2
  118. package/dist/modules/workflows/widgets/injection/order-approval/widget.client.js +11 -4
  119. package/dist/modules/workflows/widgets/injection/order-approval/widget.client.js.map +2 -2
  120. package/generated/entities/resources_resource/index.ts +1 -0
  121. package/generated/entity-fields-registry.ts +1 -0
  122. package/package.json +7 -7
  123. package/src/helpers/integration/salesFixtures.ts +34 -2
  124. package/src/helpers/integration/salesUi.ts +8 -0
  125. package/src/modules/auth/api/profile/route.ts +8 -2
  126. package/src/modules/auth/backend/auth/profile/page.tsx +1 -0
  127. package/src/modules/auth/backend/profile/change-password/page.tsx +1 -0
  128. package/src/modules/communication_channels/data/enrichers.ts +6 -26
  129. package/src/modules/customers/analytics.ts +1 -0
  130. package/src/modules/customers/components/detail/DealForm.tsx +5 -1
  131. package/src/modules/customers/components/detail/DealsSection.tsx +1 -1
  132. package/src/modules/customers/data/validators.ts +5 -1
  133. package/src/modules/dashboards/i18n/de.json +3 -0
  134. package/src/modules/dashboards/i18n/en.json +3 -0
  135. package/src/modules/dashboards/i18n/es.json +3 -0
  136. package/src/modules/dashboards/i18n/pl.json +3 -0
  137. package/src/modules/dashboards/widgets/dashboard/pipeline-summary/config.ts +86 -0
  138. package/src/modules/dashboards/widgets/dashboard/pipeline-summary/widget.client.tsx +33 -18
  139. package/src/modules/dashboards/widgets/dashboard/pipeline-summary/widget.ts +2 -2
  140. package/src/modules/data_sync/lib/adapter.ts +18 -0
  141. package/src/modules/data_sync/lib/queue-policy.ts +30 -0
  142. package/src/modules/data_sync/lib/queue.ts +19 -1
  143. package/src/modules/data_sync/lib/start-run.ts +2 -1
  144. package/src/modules/data_sync/lib/sync-engine.ts +46 -0
  145. package/src/modules/data_sync/lib/sync-run-service.ts +82 -1
  146. package/src/modules/data_sync/workers/sync-export.ts +8 -1
  147. package/src/modules/data_sync/workers/sync-import.ts +8 -1
  148. package/src/modules/dictionaries/api/[dictionaryId]/entries/route.ts +39 -11
  149. package/src/modules/dictionaries/api/openapi.ts +19 -0
  150. package/src/modules/dictionaries/components/hooks/useDictionaryEntries.ts +35 -36
  151. package/src/modules/dictionaries/data/validators.ts +14 -0
  152. package/src/modules/dictionaries/lib/clientEntries.ts +4 -3
  153. package/src/modules/dictionaries/lib/fetchAllEntries.ts +72 -0
  154. package/src/modules/entities/api/definitions.ts +51 -3
  155. package/src/modules/entities/api/entities.ts +30 -2
  156. package/src/modules/entities/api/records.ts +3 -27
  157. package/src/modules/entities/api/sidebar-entities.ts +43 -14
  158. package/src/modules/entities/lib/entityAcl.ts +58 -0
  159. package/src/modules/messages/api/route.ts +4 -5
  160. package/src/modules/messages/lib/participantScope.ts +64 -0
  161. package/src/modules/notifications/api/route.ts +2 -0
  162. package/src/modules/notifications/api/unread-count/route.ts +42 -5
  163. package/src/modules/notifications/lib/notificationScope.ts +40 -0
  164. package/src/modules/notifications/lib/notificationService.ts +5 -0
  165. package/src/modules/notifications/lib/routeHelpers.ts +19 -2
  166. package/src/modules/planner/components/unavailabilityReasons.ts +3 -3
  167. package/src/modules/progress/AGENTS.md +11 -1
  168. package/src/modules/progress/lib/progressService.ts +1 -0
  169. package/src/modules/progress/lib/progressServiceImpl.ts +385 -117
  170. package/src/modules/resources/api/resources.ts +2 -0
  171. package/src/modules/resources/backend/resources/resources/[id]/page.tsx +9 -3
  172. package/src/modules/resources/commands/resources.ts +10 -0
  173. package/src/modules/resources/components/detail/dictionaries.ts +3 -3
  174. package/src/modules/resources/data/entities.ts +3 -0
  175. package/src/modules/resources/data/validators.ts +5 -0
  176. package/src/modules/resources/lib/seeds.ts +16 -9
  177. package/src/modules/resources/migrations/.snapshot-open-mercato.json +17 -1
  178. package/src/modules/resources/migrations/Migration20260608231000.ts +13 -0
  179. package/src/modules/sales/backend/sales/documents/create/page.tsx +2 -4
  180. package/src/modules/sales/commands/documents.ts +8 -0
  181. package/src/modules/sales/components/documents/ItemsSection.tsx +19 -10
  182. package/src/modules/sales/components/documents/LineItemDialog.tsx +13 -5
  183. package/src/modules/sales/components/documents/SalesDocumentForm.tsx +48 -6
  184. package/src/modules/sales/components/documents/SalesOrderDraftLines.tsx +198 -0
  185. package/src/modules/sales/data/validators.ts +7 -1
  186. package/src/modules/sales/i18n/de.json +2 -0
  187. package/src/modules/sales/i18n/en.json +2 -0
  188. package/src/modules/sales/i18n/es.json +2 -0
  189. package/src/modules/sales/i18n/pl.json +2 -0
  190. package/src/modules/staff/components/detail/dictionaries.ts +3 -3
  191. package/src/modules/workflows/widgets/injection/order-approval/widget.client.tsx +12 -5
@@ -1,11 +1,58 @@
1
1
  import { type DateRangePreset, isValidDateRangePreset } from '@open-mercato/ui/backend/date-range'
2
+ import type { WidgetDataRequest } from '../../../services/widgetDataService'
3
+
4
+ // Deal statuses that mean the deal is closed and must stop counting as current pipeline.
5
+ //
6
+ // This is a denylist, not an allowlist: `customer_deals.status` is a lenient `text` column
7
+ // fed by the per-tenant `deal_status` dictionary, so a status unknown to code counts as OPEN
8
+ // and keeps contributing to the chart. That keeps tenant-specific stages visible and mirrors
9
+ // `customers/lib/interactionStatus.ts`, which treats an unknown interaction status as open.
10
+ //
11
+ // Every vocabulary that reaches the column has to be listed here:
12
+ // - `win` / `loose` are written by the deal closure UI and the kanban board.
13
+ // - `won` / `lost` are written verbatim by the `customers.update_deal_stage` AI tool, whose
14
+ // free-form `toStage` is passed straight through to `status`.
15
+ // - `closed` is a seeded `deal_status` dictionary value, persisted by the dashboards
16
+ // analytics seed and treated as terminal by the demo-data generator.
17
+ export const CLOSED_DEAL_STATUSES = ['win', 'loose', 'won', 'lost', 'closed'] as const
18
+
19
+ // A deal is also closed when `closure_outcome` records the outcome, which the deals-summary
20
+ // KPI already treats as terminal: `api/deals/summary/route.ts` counts a deal as won or lost
21
+ // when EITHER `status` OR `closure_outcome` says so. The two columns can disagree — the CRUD
22
+ // API accepts `closureOutcome` on its own, and the AI tool writes `status` without ever setting
23
+ // it — so the chart has to test both signals to agree with those KPI cards (#4668).
24
+ //
25
+ // The `closure_outcome` side is expressed as `IS NULL` rather than as a `neq` denylist for two
26
+ // reasons, and swapping it for `neq` would be a silent regression:
27
+ // - `closure_outcome` is nullable, and every OPEN deal holds NULL there. `neq` renders as
28
+ // `column != ?`, and in SQL `NULL != 'won'` is NULL, not true — a `neq` filter would drop
29
+ // every open deal and empty the chart. `status` has no such problem: it is `text not null
30
+ // default 'open'`, which is why the status side stays a denylist.
31
+ // - Unlike `status`, this column is not fed by a per-tenant dictionary. Every write path
32
+ // validates it against the same closed `z.enum(['won', 'lost'])` (`data/validators.ts`,
33
+ // `api/deals/[id]/route.ts`, `api/deals/[id]/stats/route.ts`), so a non-null value always
34
+ // means closed and there is no tenant-specific vocabulary to keep visible.
35
+ export const OPEN_DEAL_CLOSURE_OUTCOME_FILTER = {
36
+ field: 'closureOutcome',
37
+ operator: 'is_null',
38
+ } as const
39
+
40
+ export const PIPELINE_STATUS_SCOPES = ['open', 'all'] as const
41
+
42
+ export type PipelineStatusScope = typeof PIPELINE_STATUS_SCOPES[number]
2
43
 
3
44
  export type PipelineSummarySettings = {
4
45
  dateRange: DateRangePreset
46
+ statusScope: PipelineStatusScope
5
47
  }
6
48
 
7
49
  export const DEFAULT_SETTINGS: PipelineSummarySettings = {
8
50
  dateRange: 'this_month',
51
+ statusScope: 'open',
52
+ }
53
+
54
+ function isValidStatusScope(value: unknown): value is PipelineStatusScope {
55
+ return typeof value === 'string' && (PIPELINE_STATUS_SCOPES as readonly string[]).includes(value)
9
56
  }
10
57
 
11
58
  export function hydrateSettings(raw: unknown): PipelineSummarySettings {
@@ -13,5 +60,44 @@ export function hydrateSettings(raw: unknown): PipelineSummarySettings {
13
60
  const obj = raw as Record<string, unknown>
14
61
  return {
15
62
  dateRange: isValidDateRangePreset(obj.dateRange) ? obj.dateRange : DEFAULT_SETTINGS.dateRange,
63
+ statusScope: isValidStatusScope(obj.statusScope) ? obj.statusScope : DEFAULT_SETTINGS.statusScope,
64
+ }
65
+ }
66
+
67
+ export function dehydrateSettings(settings: PipelineSummarySettings): Record<string, unknown> {
68
+ return {
69
+ dateRange: settings.dateRange,
70
+ statusScope: settings.statusScope,
71
+ }
72
+ }
73
+
74
+ export function buildPipelineDataRequest(settings: PipelineSummarySettings): WidgetDataRequest {
75
+ const request: WidgetDataRequest = {
76
+ entityType: 'customers:deals',
77
+ metric: {
78
+ field: 'valueAmount',
79
+ aggregate: 'sum',
80
+ },
81
+ groupBy: {
82
+ field: 'pipelineStage',
83
+ resolveLabels: true,
84
+ },
85
+ dateRange: {
86
+ field: 'createdAt',
87
+ preset: settings.dateRange,
88
+ },
89
+ }
90
+
91
+ if (settings.statusScope === 'open') {
92
+ request.filters = [
93
+ ...CLOSED_DEAL_STATUSES.map((status) => ({
94
+ field: 'status',
95
+ operator: 'neq' as const,
96
+ value: status,
97
+ })),
98
+ { ...OPEN_DEAL_CLOSURE_OUTCOME_FILTER },
99
+ ]
16
100
  }
101
+
102
+ return request
17
103
  }
@@ -10,7 +10,14 @@ import {
10
10
  InlineDateRangeSelect,
11
11
  type DateRangePreset,
12
12
  } from '@open-mercato/ui/backend/date-range'
13
- import { DEFAULT_SETTINGS, hydrateSettings, type PipelineSummarySettings } from './config'
13
+ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@open-mercato/ui/primitives/select'
14
+ import {
15
+ DEFAULT_SETTINGS,
16
+ buildPipelineDataRequest,
17
+ hydrateSettings,
18
+ type PipelineStatusScope,
19
+ type PipelineSummarySettings,
20
+ } from './config'
14
21
  import type { WidgetDataResponse } from '../../../services/widgetDataService'
15
22
  import { formatCurrencyCompact } from '../../../lib/formatters'
16
23
  import { createLogger } from '@open-mercato/shared/lib/logger'
@@ -18,23 +25,7 @@ import { createLogger } from '@open-mercato/shared/lib/logger'
18
25
  const logger = createLogger('dashboards').child({ component: 'pipeline-summary' })
19
26
 
20
27
  async function fetchPipelineData(settings: PipelineSummarySettings, fetchWidgetData: WidgetDataFetcher): Promise<WidgetDataResponse> {
21
- const body = {
22
- entityType: 'customers:deals',
23
- metric: {
24
- field: 'valueAmount',
25
- aggregate: 'sum',
26
- },
27
- groupBy: {
28
- field: 'pipelineStage',
29
- resolveLabels: true,
30
- },
31
- dateRange: {
32
- field: 'createdAt',
33
- preset: settings.dateRange,
34
- },
35
- }
36
-
37
- return fetchWidgetData<WidgetDataResponse>(body)
28
+ return fetchWidgetData<WidgetDataResponse>(buildPipelineDataRequest(settings))
38
29
  }
39
30
 
40
31
  function formatStageLabel(stage: unknown, t: (key: string, fallback: string) => string): string {
@@ -97,6 +88,30 @@ const PipelineSummaryWidget: React.FC<DashboardWidgetComponentProps<PipelineSumm
97
88
  value={hydrated.dateRange}
98
89
  onChange={(dateRange: DateRangePreset) => onSettingsChange({ ...hydrated, dateRange })}
99
90
  />
91
+ <div className="space-y-1.5">
92
+ <label
93
+ htmlFor="pipeline-summary-status-scope"
94
+ className="text-xs font-semibold uppercase text-muted-foreground"
95
+ >
96
+ {t('dashboards.analytics.settings.dealStatusScope', 'Deals included')}
97
+ </label>
98
+ <Select
99
+ value={hydrated.statusScope}
100
+ onValueChange={(value) => onSettingsChange({ ...hydrated, statusScope: value as PipelineStatusScope })}
101
+ >
102
+ <SelectTrigger id="pipeline-summary-status-scope" size="sm">
103
+ <SelectValue />
104
+ </SelectTrigger>
105
+ <SelectContent>
106
+ <SelectItem value="open">
107
+ {t('dashboards.analytics.settings.dealStatusScopeOpen', 'Open deals only')}
108
+ </SelectItem>
109
+ <SelectItem value="all">
110
+ {t('dashboards.analytics.settings.dealStatusScopeAll', 'All deals, including won and lost')}
111
+ </SelectItem>
112
+ </SelectContent>
113
+ </Select>
114
+ </div>
100
115
  </div>
101
116
  )
102
117
  }
@@ -1,5 +1,5 @@
1
1
  import { lazyDashboardWidget, type DashboardWidgetModule } from '@open-mercato/shared/modules/dashboard/widgets'
2
- import { DEFAULT_SETTINGS, hydrateSettings, type PipelineSummarySettings } from './config'
2
+ import { DEFAULT_SETTINGS, dehydrateSettings, hydrateSettings, type PipelineSummarySettings } from './config'
3
3
  const PipelineSummaryWidget = lazyDashboardWidget(() => import('./widget.client'))
4
4
 
5
5
  const widget: DashboardWidgetModule<PipelineSummarySettings> = {
@@ -18,7 +18,7 @@ const widget: DashboardWidgetModule<PipelineSummarySettings> = {
18
18
  },
19
19
  Widget: PipelineSummaryWidget,
20
20
  hydrateSettings,
21
- dehydrateSettings: (s) => ({ dateRange: s.dateRange }),
21
+ dehydrateSettings,
22
22
  }
23
23
 
24
24
  export default widget
@@ -104,6 +104,24 @@ export interface DataSyncAdapter {
104
104
  readonly runMode?: 'generic' | 'provider'
105
105
  readonly operationalTelemetry?: boolean
106
106
 
107
+ /**
108
+ * Batch work MUST be replay-safe.
109
+ *
110
+ * Sync jobs are delivered at least once: BullMQ redelivers a job whose lock
111
+ * was not renewed, and the engine resumes the run from its last committed
112
+ * cursor. A batch the generator already yielded can therefore be produced and
113
+ * executed again, and the engine only fences its own commit — anything the
114
+ * generator itself did before yielding has already happened.
115
+ *
116
+ * Upserts keyed by `externalId` satisfy this. Per-record side effects that are
117
+ * not idempotent (sending mail, posting to a third party, incrementing a
118
+ * remote counter) do not, and will run twice on a resume. Make them
119
+ * conditional on state the adapter can re-read, or move them behind an event
120
+ * the engine emits after the commit.
121
+ *
122
+ * `cursor` is a resume position, not an identity: repeating it between batches
123
+ * is allowed.
124
+ */
107
125
  streamImport?(input: StreamImportInput): AsyncIterable<ImportBatch>
108
126
  streamExport?(input: StreamExportInput): AsyncIterable<ExportBatch>
109
127
  getInitialCursor?(input: { entityType: string; scope: TenantScope }): Promise<string | null>
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Queue names whose jobs resume an existing SyncRun instead of starting a fresh
3
+ * one. The workers import these to declare their `metadata.queue`, so a rename
4
+ * cannot silently drop the retry policy.
5
+ */
6
+ export const DATA_SYNC_IMPORT_QUEUE = 'data-sync-import'
7
+ export const DATA_SYNC_EXPORT_QUEUE = 'data-sync-export'
8
+
9
+ export const DATA_SYNC_RESUMABLE_QUEUES: readonly string[] = [DATA_SYNC_IMPORT_QUEUE, DATA_SYNC_EXPORT_QUEUE]
10
+
11
+ export const DATA_SYNC_QUEUE_ATTEMPTS = 3
12
+
13
+ /**
14
+ * A backfill batch can hold the processor well past BullMQ's 30s default lock,
15
+ * so these jobs stall on slow upstreams even when the worker is perfectly
16
+ * healthy. `DATA_SYNC_LOCK_DURATION_MS` is the primary defence — it stops most
17
+ * of those false stalls at the source rather than reacting to them.
18
+ *
19
+ * `DATA_SYNC_MAX_STALLED_COUNT` covers what is left. BullMQ fails a job once it
20
+ * stalls more than this many times, and at the default of 1 a single stall
21
+ * during a long backfill discards the run permanently. 10 buys enough
22
+ * redeliveries to survive a rolling restart. Raising it is only safe because a
23
+ * duplicate delivery can no longer corrupt the run: the ownership
24
+ * compare-and-swap in `commitBatchProgress` lets exactly one worker advance a
25
+ * run, and every other delivery aborts on its first commit. A job that poisons its
26
+ * worker still dies — after 10 stalls, with the run left `running` and its
27
+ * cursor unmoved, which is the state the admin runs list surfaces.
28
+ */
29
+ export const DATA_SYNC_LOCK_DURATION_MS = 120_000
30
+ export const DATA_SYNC_MAX_STALLED_COUNT = 10
@@ -1,13 +1,31 @@
1
1
  import { createModuleQueue, type Queue } from '@open-mercato/queue'
2
+ import {
3
+ DATA_SYNC_LOCK_DURATION_MS,
4
+ DATA_SYNC_MAX_STALLED_COUNT,
5
+ DATA_SYNC_QUEUE_ATTEMPTS,
6
+ DATA_SYNC_RESUMABLE_QUEUES,
7
+ } from './queue-policy'
2
8
 
3
9
  const queues = new Map<string, Queue<Record<string, unknown>>>()
4
10
 
11
+ const resumableQueueNames = new Set<string>(DATA_SYNC_RESUMABLE_QUEUES)
12
+
5
13
  export function getSyncQueue(queueName: string): Queue<Record<string, unknown>> {
6
14
  const existing = queues.get(queueName)
7
15
  if (existing) return existing
8
16
 
9
17
  const concurrency = Math.max(1, Number.parseInt(process.env.DATA_SYNC_QUEUE_CONCURRENCY ?? '5', 10) || 5)
10
- const created = createModuleQueue<Record<string, unknown>>(queueName, { concurrency })
18
+ const created = createModuleQueue<Record<string, unknown>>(
19
+ queueName,
20
+ resumableQueueNames.has(queueName)
21
+ ? {
22
+ concurrency,
23
+ attempts: DATA_SYNC_QUEUE_ATTEMPTS,
24
+ lockDuration: DATA_SYNC_LOCK_DURATION_MS,
25
+ maxStalledCount: DATA_SYNC_MAX_STALLED_COUNT,
26
+ }
27
+ : { concurrency },
28
+ )
11
29
 
12
30
  queues.set(queueName, created)
13
31
  return created
@@ -1,6 +1,7 @@
1
1
  import type { ProgressService } from '../../progress/lib/progressService'
2
2
  import type { SyncRunService } from './sync-run-service'
3
3
  import { getSyncQueue } from './queue'
4
+ import { DATA_SYNC_EXPORT_QUEUE, DATA_SYNC_IMPORT_QUEUE } from './queue-policy'
4
5
 
5
6
  export type DataSyncStartScope = {
6
7
  organizationId: string
@@ -71,7 +72,7 @@ export async function startDataSyncRun(params: {
71
72
  },
72
73
  )
73
74
 
74
- const queueName = input.direction === 'import' ? 'data-sync-import' : 'data-sync-export'
75
+ const queueName = input.direction === 'import' ? DATA_SYNC_IMPORT_QUEUE : DATA_SYNC_EXPORT_QUEUE
75
76
  const queue = getSyncQueue(queueName)
76
77
  await queue.enqueue({
77
78
  runId: run.id,
@@ -9,6 +9,7 @@ import { emitDataSyncEvent } from '../events'
9
9
  import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'
10
10
  import { getDataSyncAdapter } from './adapter-registry'
11
11
  import type { SyncRunService } from './sync-run-service'
12
+ import { SyncRunOwnershipConflictError } from './sync-run-service'
12
13
  import { createLogger } from '@open-mercato/shared/lib/logger'
13
14
 
14
15
  const logger = createLogger('data_sync').child({ component: 'sync-engine' })
@@ -236,6 +237,21 @@ export function createSyncEngine(deps: EngineDeps) {
236
237
  return
237
238
  }
238
239
 
240
+ if (run.status !== status) {
241
+ // `markStatus` refuses a terminal -> different-terminal transition and
242
+ // returns the row unchanged, so the run is already finished under another
243
+ // delivery of this job. Everything below — the progress job, the
244
+ // operational log and the lifecycle event — would describe the wrong
245
+ // outcome, and `data_sync.run.failed` is dispatched to tenant webhooks.
246
+ // A displaced worker stays silent instead.
247
+ logger.warn('Skipping finalization of a sync run another worker already finalized', {
248
+ runId,
249
+ requestedStatus: status,
250
+ actualStatus: run.status,
251
+ })
252
+ return
253
+ }
254
+
239
255
  if (run.progressJobId) {
240
256
  if (status === 'completed') {
241
257
  await progressService.completeJob(
@@ -408,6 +424,10 @@ export function createSyncEngine(deps: EngineDeps) {
408
424
  throw new Error(`Integration ${run.integrationId} is missing credentials`)
409
425
  }
410
426
 
427
+ // A run already `running` means a stalled job was redelivered, so this is
428
+ // a resume rather than a first start. Consumers see one `started` event per
429
+ // delivery either way; the flag is what lets them tell the two apart.
430
+ const resumed = run.status === 'running'
411
431
  const activeRun = await syncRunService.markStatus(run.id, 'running', scope)
412
432
  if (!activeRun || activeRun.status !== 'running') {
413
433
  return
@@ -417,6 +437,7 @@ export function createSyncEngine(deps: EngineDeps) {
417
437
  integrationId: run.integrationId,
418
438
  entityType: run.entityType,
419
439
  direction: run.direction,
440
+ resumed,
420
441
  tenantId: scope.tenantId,
421
442
  organizationId: scope.organizationId,
422
443
  })
@@ -452,6 +473,7 @@ export function createSyncEngine(deps: EngineDeps) {
452
473
  const mapping = await resolveMapping(adapter, run.entityType, scope)
453
474
  let processedCount = 0
454
475
  let totalCount: number | null = null
476
+ let committedBatches = activeRun.batchesCompleted ?? 0
455
477
 
456
478
  try {
457
479
  for await (const batch of adapter.streamImport({
@@ -481,7 +503,9 @@ export function createSyncEngine(deps: EngineDeps) {
481
503
  },
482
504
  batch.cursor,
483
505
  scope,
506
+ committedBatches,
484
507
  )
508
+ committedBatches += 1
485
509
 
486
510
  await updateProgress(run.progressJobId, processedCount, totalCount, scope)
487
511
  await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)
@@ -507,6 +531,13 @@ export function createSyncEngine(deps: EngineDeps) {
507
531
  })
508
532
  }
509
533
  } catch (error) {
534
+ if (error instanceof SyncRunOwnershipConflictError) {
535
+ logger.warn('Yielding import run to a concurrent worker that already advanced it', {
536
+ runId: run.id,
537
+ expectedBatchesCompleted: error.expectedBatchesCompleted,
538
+ })
539
+ return
540
+ }
510
541
  const message = error instanceof Error ? error.message : 'Sync import failed'
511
542
  await integrationLogService.write(
512
543
  {
@@ -553,6 +584,10 @@ export function createSyncEngine(deps: EngineDeps) {
553
584
  throw new Error(`Integration ${run.integrationId} is missing credentials`)
554
585
  }
555
586
 
587
+ // A run already `running` means a stalled job was redelivered, so this is
588
+ // a resume rather than a first start. Consumers see one `started` event per
589
+ // delivery either way; the flag is what lets them tell the two apart.
590
+ const resumed = run.status === 'running'
556
591
  const activeRun = await syncRunService.markStatus(run.id, 'running', scope)
557
592
  if (!activeRun || activeRun.status !== 'running') {
558
593
  return
@@ -562,6 +597,7 @@ export function createSyncEngine(deps: EngineDeps) {
562
597
  integrationId: run.integrationId,
563
598
  entityType: run.entityType,
564
599
  direction: run.direction,
600
+ resumed,
565
601
  tenantId: scope.tenantId,
566
602
  organizationId: scope.organizationId,
567
603
  })
@@ -596,6 +632,7 @@ export function createSyncEngine(deps: EngineDeps) {
596
632
 
597
633
  const mapping = await resolveMapping(adapter, run.entityType, scope)
598
634
  let processedCount = 0
635
+ let committedBatches = activeRun.batchesCompleted ?? 0
599
636
 
600
637
  try {
601
638
  for await (const batch of adapter.streamExport({
@@ -626,7 +663,9 @@ export function createSyncEngine(deps: EngineDeps) {
626
663
  },
627
664
  batch.cursor,
628
665
  scope,
666
+ committedBatches,
629
667
  )
668
+ committedBatches += 1
630
669
  await updateProgress(run.progressJobId, processedCount, null, scope)
631
670
  await logExportItemFailures(run.id, run.integrationId, batch.results, scope)
632
671
 
@@ -647,6 +686,13 @@ export function createSyncEngine(deps: EngineDeps) {
647
686
  })
648
687
  }
649
688
  } catch (error) {
689
+ if (error instanceof SyncRunOwnershipConflictError) {
690
+ logger.warn('Yielding export run to a concurrent worker that already advanced it', {
691
+ runId: run.id,
692
+ expectedBatchesCompleted: error.expectedBatchesCompleted,
693
+ })
694
+ return
695
+ }
650
696
  const message = error instanceof Error ? error.message : 'Sync export failed'
651
697
  await integrationLogService.write(
652
698
  {
@@ -26,6 +26,27 @@ type SyncScope = {
26
26
  tenantId: string
27
27
  }
28
28
 
29
+ /**
30
+ * Raised when a batch commit loses the ownership compare-and-swap, meaning
31
+ * another delivery of the same job advanced the run while this worker was
32
+ * streaming.
33
+ *
34
+ * BullMQ guarantees at-least-once delivery: a job whose lock is not renewed is
35
+ * redelivered under the SAME job id, whether the previous worker died or is only
36
+ * blocked. No identity token can tell those apart, so ownership is enforced here
37
+ * — on the write that matters — instead of at claim time. The loser aborts and
38
+ * leaves the run to the worker that is still making progress.
39
+ */
40
+ export class SyncRunOwnershipConflictError extends Error {
41
+ constructor(
42
+ readonly runId: string,
43
+ readonly expectedBatchesCompleted: number,
44
+ ) {
45
+ super(`[internal] Sync run ${runId} advanced past batch ${expectedBatchesCompleted} under a concurrent worker`)
46
+ this.name = 'SyncRunOwnershipConflictError'
47
+ }
48
+ }
49
+
29
50
  export function createSyncRunService(em: EntityManager) {
30
51
  async function resolveCursorRow(run: SyncRun, scope: SyncScope): Promise<SyncCursor | null> {
31
52
  return findOneWithDecryption(
@@ -150,7 +171,11 @@ export function createSyncRunService(em: EntityManager) {
150
171
  organizationId: scope.organizationId,
151
172
  tenantId: scope.tenantId,
152
173
  deletedAt: null,
153
- status: 'pending',
174
+ // A BullMQ stalled-job redelivery finds the run in `running` after
175
+ // the previous worker was hard-killed. Treat that transition as an
176
+ // idempotent claim while still excluding terminal states so a
177
+ // cancelled or completed run cannot be revived.
178
+ status: { $in: ['pending', 'running'] },
154
179
  },
155
180
  {
156
181
  status,
@@ -178,6 +203,12 @@ export function createSyncRunService(em: EntityManager) {
178
203
  return row
179
204
  },
180
205
 
206
+ /**
207
+ * @deprecated Use {@link commitBatchProgress}, which writes counters and
208
+ * cursor in one transaction behind the ownership fence. This method updates
209
+ * counters unfenced, so two deliveries of the same job can lose each other's
210
+ * increments. Kept for external callers only.
211
+ */
181
212
  async updateCounts(
182
213
  runId: string,
183
214
  delta: Partial<Pick<SyncRun, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount' | 'batchesCompleted'>>,
@@ -195,6 +226,11 @@ export function createSyncRunService(em: EntityManager) {
195
226
  return row
196
227
  },
197
228
 
229
+ /**
230
+ * @deprecated Use {@link commitBatchProgress}. This method advances the
231
+ * cursor without the ownership fence, so a stale delivery can move the
232
+ * cursor of a run another worker owns. Kept for external callers only.
233
+ */
198
234
  async updateCursor(runId: string, cursor: string, scope: SyncScope): Promise<void> {
199
235
  const run = await this.getRun(runId, scope)
200
236
  if (!run) return
@@ -204,16 +240,61 @@ export function createSyncRunService(em: EntityManager) {
204
240
  ], { transaction: true })
205
241
  },
206
242
 
243
+ /**
244
+ * Commits one batch's counters and cursor in a single transaction.
245
+ *
246
+ * Passing `expectedBatchesCompleted` fences the write: the run must still be
247
+ * `running` and still sit on that batch count, or another delivery of the
248
+ * same BullMQ job owns the run and this commit throws
249
+ * `SyncRunOwnershipConflictError` and rolls back. Omitting it keeps the
250
+ * legacy unguarded write for callers outside the engine.
251
+ *
252
+ * The fence token is `batchesCompleted` rather than `cursor` because it
253
+ * advances by construction on every commit. A cursor is a free-form adapter
254
+ * string that an adapter may legitimately repeat between batches — the
255
+ * Akeneo products adapter does, between its final page and the
256
+ * reconciliation batch that follows it — and a repeated token fences
257
+ * nothing.
258
+ *
259
+ * The guard's `UPDATE` also holds the row lock for the rest of the
260
+ * transaction, so a competing commit blocks here and then re-reads the
261
+ * advanced count instead of interleaving with this one. That is what lets
262
+ * the counters below stay a plain read-modify-write against the snapshot
263
+ * read above: a commit that wins the fence has proven that nothing else
264
+ * landed since it read.
265
+ */
207
266
  async commitBatchProgress(
208
267
  runId: string,
209
268
  delta: Partial<Pick<SyncRun, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount' | 'batchesCompleted'>>,
210
269
  cursor: string,
211
270
  scope: SyncScope,
271
+ expectedBatchesCompleted?: number,
212
272
  ): Promise<SyncRun | null> {
213
273
  const run = await this.getRun(runId, scope)
214
274
  if (!run) return null
215
275
  const cursorRow = await resolveCursorRow(run, scope)
276
+ const claimRunOwnership = async () => {
277
+ if ((delta.batchesCompleted ?? 0) < 1) {
278
+ throw new Error(`[internal] A fenced commit for sync run ${runId} must advance batchesCompleted`)
279
+ }
280
+ const owned = await em.nativeUpdate(
281
+ SyncRun,
282
+ {
283
+ id: runId,
284
+ organizationId: scope.organizationId,
285
+ tenantId: scope.tenantId,
286
+ deletedAt: null,
287
+ status: 'running',
288
+ batchesCompleted: expectedBatchesCompleted,
289
+ },
290
+ { updatedAt: new Date() },
291
+ )
292
+ if (owned === 0) {
293
+ throw new SyncRunOwnershipConflictError(runId, expectedBatchesCompleted ?? 0)
294
+ }
295
+ }
216
296
  await withAtomicFlush(em, [
297
+ ...(expectedBatchesCompleted === undefined ? [] : [claimRunOwnership]),
217
298
  () => {
218
299
  run.createdCount += delta.createdCount ?? 0
219
300
  run.updatedCount += delta.updatedCount ?? 0
@@ -2,6 +2,11 @@ import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
2
2
  import type { ProgressService } from '../../progress/lib/progressService'
3
3
  import type { SyncEngine } from '../lib/sync-engine'
4
4
  import type { SyncRunService } from '../lib/sync-run-service'
5
+ import {
6
+ DATA_SYNC_EXPORT_QUEUE,
7
+ DATA_SYNC_LOCK_DURATION_MS,
8
+ DATA_SYNC_MAX_STALLED_COUNT,
9
+ } from '../lib/queue-policy'
5
10
  import { createLogger } from '@open-mercato/shared/lib/logger'
6
11
 
7
12
  const logger = createLogger('data_sync').child({ component: 'sync-export' })
@@ -17,9 +22,11 @@ type SyncJobPayload = {
17
22
  }
18
23
 
19
24
  export const metadata: WorkerMeta = {
20
- queue: 'data-sync-export',
25
+ queue: DATA_SYNC_EXPORT_QUEUE,
21
26
  id: 'data-sync:export',
22
27
  concurrency: 5,
28
+ lockDuration: DATA_SYNC_LOCK_DURATION_MS,
29
+ maxStalledCount: DATA_SYNC_MAX_STALLED_COUNT,
23
30
  }
24
31
 
25
32
  type HandlerContext = JobContext & {
@@ -2,6 +2,11 @@ import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
2
2
  import type { ProgressService } from '../../progress/lib/progressService'
3
3
  import type { SyncEngine } from '../lib/sync-engine'
4
4
  import type { SyncRunService } from '../lib/sync-run-service'
5
+ import {
6
+ DATA_SYNC_IMPORT_QUEUE,
7
+ DATA_SYNC_LOCK_DURATION_MS,
8
+ DATA_SYNC_MAX_STALLED_COUNT,
9
+ } from '../lib/queue-policy'
5
10
  import { createLogger } from '@open-mercato/shared/lib/logger'
6
11
 
7
12
  const logger = createLogger('data_sync').child({ component: 'sync-import' })
@@ -17,9 +22,11 @@ type SyncJobPayload = {
17
22
  }
18
23
 
19
24
  export const metadata: WorkerMeta = {
20
- queue: 'data-sync-import',
25
+ queue: DATA_SYNC_IMPORT_QUEUE,
21
26
  id: 'data-sync:import',
22
27
  concurrency: 5,
28
+ lockDuration: DATA_SYNC_LOCK_DURATION_MS,
29
+ maxStalledCount: DATA_SYNC_MAX_STALLED_COUNT,
23
30
  }
24
31
 
25
32
  type HandlerContext = JobContext & {