@open-mercato/core 0.6.8-develop.6987.1.02f619470c → 0.6.8-develop.6992.1.00c90fecff
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/customers/components/detail/AssignRoleDialog.js +9 -3
- package/dist/modules/customers/components/detail/AssignRoleDialog.js.map +2 -2
- package/dist/modules/customers/components/detail/DealsSection.js +2 -3
- package/dist/modules/customers/components/detail/DealsSection.js.map +2 -2
- package/dist/modules/customers/components/detail/assignableStaff.js +2 -1
- package/dist/modules/customers/components/detail/assignableStaff.js.map +2 -2
- package/dist/modules/customers/components/detail/schedule/LinkedEntitiesField.js +11 -12
- package/dist/modules/customers/components/detail/schedule/LinkedEntitiesField.js.map +2 -2
- package/dist/modules/customers/components/detail/schedule/ParticipantsField.js +7 -4
- package/dist/modules/customers/components/detail/schedule/ParticipantsField.js.map +2 -2
- package/dist/modules/data_sync/api/run.js +9 -1
- package/dist/modules/data_sync/api/run.js.map +2 -2
- package/dist/modules/data_sync/api/runs/[id]/retry.js +9 -1
- package/dist/modules/data_sync/api/runs/[id]/retry.js.map +2 -2
- package/dist/modules/data_sync/lib/adapter-registry.js +10 -1
- package/dist/modules/data_sync/lib/adapter-registry.js.map +2 -2
- package/dist/modules/data_sync/lib/start-cursor.js +17 -0
- package/dist/modules/data_sync/lib/start-cursor.js.map +7 -0
- package/dist/modules/data_sync/lib/sync-engine.js +5 -7
- package/dist/modules/data_sync/lib/sync-engine.js.map +2 -2
- package/dist/modules/data_sync/lib/sync-run-service.js +86 -10
- package/dist/modules/data_sync/lib/sync-run-service.js.map +2 -2
- package/dist/modules/data_sync/workers/sync-scheduled.js +9 -6
- package/dist/modules/data_sync/workers/sync-scheduled.js.map +2 -2
- package/dist/modules/sales/api/channels/route.js +1 -1
- package/dist/modules/sales/api/channels/route.js.map +2 -2
- package/package.json +7 -7
- package/src/modules/customers/components/detail/AssignRoleDialog.tsx +12 -3
- package/src/modules/customers/components/detail/DealsSection.tsx +6 -11
- package/src/modules/customers/components/detail/assignableStaff.ts +9 -1
- package/src/modules/customers/components/detail/schedule/LinkedEntitiesField.tsx +16 -13
- package/src/modules/customers/components/detail/schedule/ParticipantsField.tsx +11 -4
- package/src/modules/data_sync/AGENTS.md +4 -1
- package/src/modules/data_sync/api/run.ts +9 -1
- package/src/modules/data_sync/api/runs/[id]/retry.ts +9 -1
- package/src/modules/data_sync/lib/adapter-registry.ts +15 -0
- package/src/modules/data_sync/lib/adapter.ts +18 -0
- package/src/modules/data_sync/lib/start-cursor.ts +35 -0
- package/src/modules/data_sync/lib/sync-engine.ts +5 -7
- package/src/modules/data_sync/lib/sync-run-service.ts +118 -10
- package/src/modules/data_sync/workers/sync-scheduled.ts +9 -6
- package/src/modules/sales/api/channels/route.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/core",
|
|
3
|
-
"version": "0.6.8-develop.
|
|
3
|
+
"version": "0.6.8-develop.6992.1.00c90fecff",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -254,16 +254,16 @@
|
|
|
254
254
|
"zod": "^4.4.3"
|
|
255
255
|
},
|
|
256
256
|
"peerDependencies": {
|
|
257
|
-
"@open-mercato/ai-assistant": "0.6.8-develop.
|
|
258
|
-
"@open-mercato/shared": "0.6.8-develop.
|
|
259
|
-
"@open-mercato/ui": "0.6.8-develop.
|
|
257
|
+
"@open-mercato/ai-assistant": "0.6.8-develop.6992.1.00c90fecff",
|
|
258
|
+
"@open-mercato/shared": "0.6.8-develop.6992.1.00c90fecff",
|
|
259
|
+
"@open-mercato/ui": "0.6.8-develop.6992.1.00c90fecff",
|
|
260
260
|
"react": "^19.0.0",
|
|
261
261
|
"react-dom": "^19.0.0"
|
|
262
262
|
},
|
|
263
263
|
"devDependencies": {
|
|
264
|
-
"@open-mercato/ai-assistant": "0.6.8-develop.
|
|
265
|
-
"@open-mercato/shared": "0.6.8-develop.
|
|
266
|
-
"@open-mercato/ui": "0.6.8-develop.
|
|
264
|
+
"@open-mercato/ai-assistant": "0.6.8-develop.6992.1.00c90fecff",
|
|
265
|
+
"@open-mercato/shared": "0.6.8-develop.6992.1.00c90fecff",
|
|
266
|
+
"@open-mercato/ui": "0.6.8-develop.6992.1.00c90fecff",
|
|
267
267
|
"@testing-library/dom": "^10.4.1",
|
|
268
268
|
"@testing-library/jest-dom": "^7.0.0",
|
|
269
269
|
"@testing-library/react": "^16.3.1",
|
|
@@ -7,6 +7,7 @@ import { Avatar } from '@open-mercato/ui/primitives/avatar'
|
|
|
7
7
|
import { EmptyState } from '@open-mercato/ui/primitives/empty-state'
|
|
8
8
|
import { StepIndicator, type StepIndicatorStep } from '@open-mercato/ui/primitives/step-indicator'
|
|
9
9
|
import { useT } from '@open-mercato/shared/lib/i18n/context'
|
|
10
|
+
import { hasMoreFromPage } from '@open-mercato/shared/lib/pagination/load-more'
|
|
10
11
|
import { Button } from '@open-mercato/ui/primitives/button'
|
|
11
12
|
import { Badge } from '@open-mercato/ui/primitives/badge'
|
|
12
13
|
import { Input } from '@open-mercato/ui/primitives/input'
|
|
@@ -75,6 +76,10 @@ export function AssignRoleDialog({
|
|
|
75
76
|
const [activeTeam, setActiveTeam] = React.useState('all')
|
|
76
77
|
const [loadError, setLoadError] = React.useState<string | null>(null)
|
|
77
78
|
const [totalUsers, setTotalUsers] = React.useState(0)
|
|
79
|
+
// Short-page termination instead of `users.length >= totalUsers` — see
|
|
80
|
+
// `hasMoreFromPage`. Measured on the served count, not on `users`, which is
|
|
81
|
+
// deduped by id.
|
|
82
|
+
const [hasMore, setHasMore] = React.useState(false)
|
|
78
83
|
const [currentPage, setCurrentPage] = React.useState(1)
|
|
79
84
|
const deferredSearchQuery = React.useDeferredValue(searchQuery)
|
|
80
85
|
const requestSequenceRef = React.useRef(0)
|
|
@@ -94,6 +99,7 @@ export function AssignRoleDialog({
|
|
|
94
99
|
setActiveTeam('all')
|
|
95
100
|
setLoadError(null)
|
|
96
101
|
setTotalUsers(0)
|
|
102
|
+
setHasMore(false)
|
|
97
103
|
setCurrentPage(1)
|
|
98
104
|
requestSequenceRef.current = 0
|
|
99
105
|
return
|
|
@@ -110,6 +116,7 @@ export function AssignRoleDialog({
|
|
|
110
116
|
setActiveTeam('all')
|
|
111
117
|
setLoadError(null)
|
|
112
118
|
setTotalUsers(0)
|
|
119
|
+
setHasMore(false)
|
|
113
120
|
setCurrentPage(1)
|
|
114
121
|
requestSequenceRef.current = 0
|
|
115
122
|
}, [initialRoleType, open])
|
|
@@ -154,6 +161,7 @@ export function AssignRoleDialog({
|
|
|
154
161
|
return Array.from(merged.values())
|
|
155
162
|
})
|
|
156
163
|
setTotalUsers(result.total)
|
|
164
|
+
setHasMore(hasMoreFromPage(result.servedCount, ASSIGNABLE_STAFF_PAGE_SIZE))
|
|
157
165
|
setCurrentPage(result.page)
|
|
158
166
|
setLoadError(null)
|
|
159
167
|
} catch {
|
|
@@ -161,6 +169,7 @@ export function AssignRoleDialog({
|
|
|
161
169
|
if (!append) {
|
|
162
170
|
setUsers([])
|
|
163
171
|
setTotalUsers(0)
|
|
172
|
+
setHasMore(false)
|
|
164
173
|
setCurrentPage(1)
|
|
165
174
|
}
|
|
166
175
|
setLoadError(
|
|
@@ -189,14 +198,14 @@ export function AssignRoleDialog({
|
|
|
189
198
|
}, [deferredSearchQuery, searchUsers, step])
|
|
190
199
|
|
|
191
200
|
const handleLoadMore = React.useCallback(() => {
|
|
192
|
-
if (loading || loadingMore ||
|
|
201
|
+
if (loading || loadingMore || !hasMore) return
|
|
193
202
|
// fire-and-forget: search results populate async; errors shown in list UI
|
|
194
203
|
searchUsers({
|
|
195
204
|
query: deferredSearchQuery,
|
|
196
205
|
page: currentPage + 1,
|
|
197
206
|
append: true,
|
|
198
207
|
}).catch(() => {})
|
|
199
|
-
}, [currentPage, deferredSearchQuery, loading, loadingMore, searchUsers
|
|
208
|
+
}, [currentPage, deferredSearchQuery, hasMore, loading, loadingMore, searchUsers])
|
|
200
209
|
|
|
201
210
|
const selectedRole = React.useMemo(
|
|
202
211
|
() => roleTypes.find((roleType) => roleType.value === selectedRoleType) ?? null,
|
|
@@ -262,7 +271,7 @@ export function AssignRoleDialog({
|
|
|
262
271
|
)
|
|
263
272
|
}, [t, totalUsers, users.length])
|
|
264
273
|
|
|
265
|
-
const canLoadMore =
|
|
274
|
+
const canLoadMore = hasMore
|
|
266
275
|
|
|
267
276
|
const handleAssign = React.useCallback(async () => {
|
|
268
277
|
if (!selectedRoleType || !selectedUser) return
|
|
@@ -13,6 +13,7 @@ import { createDealLinkAdapter } from '../linking/adapters/dealAdapter'
|
|
|
13
13
|
import { LoadingMessage, TabEmptyState } from '@open-mercato/ui/backend/detail'
|
|
14
14
|
import { useOrganizationScopeVersion } from '@open-mercato/shared/lib/frontend/useOrganizationScope'
|
|
15
15
|
import { useT } from '@open-mercato/shared/lib/i18n/context'
|
|
16
|
+
import { hasMoreFromPage } from '@open-mercato/shared/lib/pagination/load-more'
|
|
16
17
|
import { useConfirmDialog } from '@open-mercato/ui/backend/confirm-dialog'
|
|
17
18
|
import { E } from '#generated/entities.ids.generated'
|
|
18
19
|
import type { DealCustomFieldEntry, DealSummary, SectionAction, TabEmptyStateConfig, Translator } from './types'
|
|
@@ -545,17 +546,11 @@ export function DealsSection({
|
|
|
545
546
|
return [...updatedPrev, ...appended]
|
|
546
547
|
})
|
|
547
548
|
pageRef.current = nextPage
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
? Number(totalPagesRaw)
|
|
554
|
-
: null
|
|
555
|
-
const nextHasMore =
|
|
556
|
-
totalPages && Number.isFinite(totalPages)
|
|
557
|
-
? nextPage < totalPages
|
|
558
|
-
: mapped.length === DEALS_PAGE_SIZE
|
|
549
|
+
// Short-page termination, unconditionally — see `hasMoreFromPage`.
|
|
550
|
+
// Preferring `totalPages` here hid deals that do exist. Measured on
|
|
551
|
+
// `rawItems`, which is what the server served, rather than on the
|
|
552
|
+
// deduped rows appended above.
|
|
553
|
+
const nextHasMore = hasMoreFromPage(rawItems.length, DEALS_PAGE_SIZE)
|
|
559
554
|
hasMoreRef.current = nextHasMore
|
|
560
555
|
setHasMore(nextHasMore)
|
|
561
556
|
setLoadError(null)
|
|
@@ -18,6 +18,13 @@ type AssignableStaffResponse = {
|
|
|
18
18
|
|
|
19
19
|
export type AssignableStaffMembersPage = {
|
|
20
20
|
items: AssignableStaffMember[]
|
|
21
|
+
/**
|
|
22
|
+
* How many rows the server served for this page, before the dedupe below.
|
|
23
|
+
* "Load more" guards must measure this rather than `items.length`: a deduped
|
|
24
|
+
* length shorter than the served page reads as a short page and terminates
|
|
25
|
+
* the sequence early, stranding the rest of the roster.
|
|
26
|
+
*/
|
|
27
|
+
servedCount: number
|
|
21
28
|
total: number
|
|
22
29
|
page: number
|
|
23
30
|
pageSize: number
|
|
@@ -59,7 +66,7 @@ export async function fetchAssignableStaffMembersPage(
|
|
|
59
66
|
)
|
|
60
67
|
} catch (error) {
|
|
61
68
|
if (isAssignableEndpointMissing(error)) {
|
|
62
|
-
return { items: [], total: 0, page, pageSize }
|
|
69
|
+
return { items: [], servedCount: 0, total: 0, page, pageSize }
|
|
63
70
|
}
|
|
64
71
|
throw error
|
|
65
72
|
}
|
|
@@ -125,6 +132,7 @@ export async function fetchAssignableStaffMembersPage(
|
|
|
125
132
|
|
|
126
133
|
return {
|
|
127
134
|
items: Array.from(deduped.values()),
|
|
135
|
+
servedCount: rawItems.length,
|
|
128
136
|
total:
|
|
129
137
|
typeof data?.total === 'number' && Number.isFinite(data.total)
|
|
130
138
|
? data.total
|
|
@@ -4,6 +4,7 @@ import * as React from 'react'
|
|
|
4
4
|
import { Building2, Briefcase, FileText, Search, X } from 'lucide-react'
|
|
5
5
|
import { cn } from '@open-mercato/shared/lib/utils'
|
|
6
6
|
import { useT } from '@open-mercato/shared/lib/i18n/context'
|
|
7
|
+
import { hasMoreFromPage } from '@open-mercato/shared/lib/pagination/load-more'
|
|
7
8
|
import { readApiResultOrThrow } from '@open-mercato/ui/backend/utils/apiCall'
|
|
8
9
|
import { Button } from '@open-mercato/ui/primitives/button'
|
|
9
10
|
import { IconButton } from '@open-mercato/ui/primitives/icon-button'
|
|
@@ -14,6 +15,8 @@ import type { LinkedEntity } from './useScheduleFormState'
|
|
|
14
15
|
|
|
15
16
|
const ENTITY_LINK_TYPES = ['company', 'deal', 'offer'] as const
|
|
16
17
|
|
|
18
|
+
const PAGE_SIZE = 20
|
|
19
|
+
|
|
17
20
|
function readLabelCandidate(value: unknown): string | null {
|
|
18
21
|
if (typeof value !== 'string') return null
|
|
19
22
|
const trimmed = value.trim()
|
|
@@ -78,7 +81,10 @@ function EntityLinkSearchPopover({
|
|
|
78
81
|
const [query, setQuery] = React.useState('')
|
|
79
82
|
const [results, setResults] = React.useState<Array<{ id: string; label: string }>>([])
|
|
80
83
|
const [page, setPage] = React.useState(1)
|
|
81
|
-
|
|
84
|
+
// Short-page termination instead of a `total`/`totalPages` bound — see
|
|
85
|
+
// `hasMoreFromPage`. `items` is the raw response, before the mapping and
|
|
86
|
+
// filtering below, so its length is what the server served.
|
|
87
|
+
const [hasMore, setHasMore] = React.useState(false)
|
|
82
88
|
const [loading, setLoading] = React.useState(false)
|
|
83
89
|
const selectableResults = React.useMemo(
|
|
84
90
|
() => results.filter((result) => !existingIds.has(result.id)),
|
|
@@ -90,12 +96,12 @@ function EntityLinkSearchPopover({
|
|
|
90
96
|
const controller = new AbortController()
|
|
91
97
|
setLoading(true)
|
|
92
98
|
const searchParam = query.trim() ? `&search=${encodeURIComponent(query.trim())}` : ''
|
|
93
|
-
const pagingParam = `&page=${page}&pageSize
|
|
99
|
+
const pagingParam = `&page=${page}&pageSize=${PAGE_SIZE}`
|
|
94
100
|
const endpoint = linkType === 'company'
|
|
95
101
|
? `/api/customers/companies?sortField=name&sortDir=asc${pagingParam}${searchParam}`
|
|
96
102
|
: linkType === 'deal'
|
|
97
|
-
? `/api/customers/deals?pageSize
|
|
98
|
-
: `/api/sales/quotes?pageSize
|
|
103
|
+
? `/api/customers/deals?pageSize=${PAGE_SIZE}&page=${page}${searchParam}`
|
|
104
|
+
: `/api/sales/quotes?pageSize=${PAGE_SIZE}&page=${page}${searchParam}`
|
|
99
105
|
readApiResultOrThrow<{ items?: Array<Record<string, unknown>>; totalPages?: number; page?: number; pageSize?: number; total?: number }>(endpoint, { signal: controller.signal })
|
|
100
106
|
.then((data) => {
|
|
101
107
|
const items = Array.isArray(data?.items) ? data.items : []
|
|
@@ -109,15 +115,12 @@ function EntityLinkSearchPopover({
|
|
|
109
115
|
nextResults.forEach((entry) => merged.set(entry.id, entry))
|
|
110
116
|
return Array.from(merged.values())
|
|
111
117
|
})
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
setTotalPages(1)
|
|
118
|
-
}
|
|
118
|
+
setHasMore(hasMoreFromPage(items.length, PAGE_SIZE))
|
|
119
|
+
})
|
|
120
|
+
.catch(() => {
|
|
121
|
+
setResults([])
|
|
122
|
+
setHasMore(false)
|
|
119
123
|
})
|
|
120
|
-
.catch(() => setResults([]))
|
|
121
124
|
.finally(() => setLoading(false))
|
|
122
125
|
return () => controller.abort()
|
|
123
126
|
}, [open, page, query, linkType])
|
|
@@ -212,7 +215,7 @@ function EntityLinkSearchPopover({
|
|
|
212
215
|
</Button>
|
|
213
216
|
)
|
|
214
217
|
})}
|
|
215
|
-
{!loading &&
|
|
218
|
+
{!loading && hasMore ? (
|
|
216
219
|
<div className="px-2 py-2">
|
|
217
220
|
<Button type="button" variant="outline" size="sm" className="w-full" onClick={() => setPage((current) => current + 1)}>
|
|
218
221
|
{t('customers.schedule.loadMore', 'Load more')}
|
|
@@ -4,6 +4,7 @@ import * as React from 'react'
|
|
|
4
4
|
import { Users, X, Clock, CheckCircle2, XCircle } from 'lucide-react'
|
|
5
5
|
import { cn } from '@open-mercato/shared/lib/utils'
|
|
6
6
|
import { useT } from '@open-mercato/shared/lib/i18n/context'
|
|
7
|
+
import { hasMoreFromPage } from '@open-mercato/shared/lib/pagination/load-more'
|
|
7
8
|
import { Button } from '@open-mercato/ui/primitives/button'
|
|
8
9
|
import { Checkbox } from '@open-mercato/ui/primitives/checkbox'
|
|
9
10
|
import { IconButton } from '@open-mercato/ui/primitives/icon-button'
|
|
@@ -15,6 +16,8 @@ import { isVisible, getFieldLabel } from './fieldConfig'
|
|
|
15
16
|
import type { Participant, RsvpStatus } from './useScheduleFormState'
|
|
16
17
|
import { PARTICIPANT_COLORS } from './useScheduleFormState'
|
|
17
18
|
|
|
19
|
+
const PAGE_SIZE = 20
|
|
20
|
+
|
|
18
21
|
function ParticipantSearchPopover({
|
|
19
22
|
existingIds,
|
|
20
23
|
onAdd,
|
|
@@ -30,7 +33,10 @@ function ParticipantSearchPopover({
|
|
|
30
33
|
const [query, setQuery] = React.useState('')
|
|
31
34
|
const [results, setResults] = React.useState<Array<{ userId: string; name: string; email: string }>>([])
|
|
32
35
|
const [page, setPage] = React.useState(1)
|
|
33
|
-
|
|
36
|
+
// Short-page termination instead of a `total`-derived page bound — see
|
|
37
|
+
// `hasMoreFromPage`. Measured on the served count, not on `members`, which is
|
|
38
|
+
// deduped by user id.
|
|
39
|
+
const [hasMore, setHasMore] = React.useState(false)
|
|
34
40
|
const [loading, setLoading] = React.useState(false)
|
|
35
41
|
const [loadError, setLoadError] = React.useState<string | null>(null)
|
|
36
42
|
const selectableResults = React.useMemo(
|
|
@@ -42,7 +48,7 @@ function ParticipantSearchPopover({
|
|
|
42
48
|
if (!open) return
|
|
43
49
|
const controller = new AbortController()
|
|
44
50
|
setLoading(true)
|
|
45
|
-
fetchAssignableStaffMembersPage(query, { page, pageSize:
|
|
51
|
+
fetchAssignableStaffMembersPage(query, { page, pageSize: PAGE_SIZE, signal: controller.signal })
|
|
46
52
|
.then((result) => {
|
|
47
53
|
const members = result.items
|
|
48
54
|
const nextResults = members.map((member) => ({
|
|
@@ -56,11 +62,12 @@ function ParticipantSearchPopover({
|
|
|
56
62
|
nextResults.forEach((entry) => merged.set(entry.userId, entry))
|
|
57
63
|
return Array.from(merged.values())
|
|
58
64
|
})
|
|
59
|
-
|
|
65
|
+
setHasMore(hasMoreFromPage(result.servedCount, PAGE_SIZE))
|
|
60
66
|
setLoadError(null)
|
|
61
67
|
})
|
|
62
68
|
.catch(() => {
|
|
63
69
|
setResults([])
|
|
70
|
+
setHasMore(false)
|
|
64
71
|
setLoadError(
|
|
65
72
|
t(
|
|
66
73
|
'customers.assignableStaff.loadError',
|
|
@@ -148,7 +155,7 @@ function ParticipantSearchPopover({
|
|
|
148
155
|
</Button>
|
|
149
156
|
)
|
|
150
157
|
})}
|
|
151
|
-
{!loading && !loadError &&
|
|
158
|
+
{!loading && !loadError && hasMore ? (
|
|
152
159
|
<div className="px-2 py-2">
|
|
153
160
|
<Button type="button" variant="outline" size="sm" className="w-full" onClick={() => setPage((current) => current + 1)}>
|
|
154
161
|
{t('customers.schedule.loadMore', 'Load more')}
|
|
@@ -104,6 +104,7 @@ interface DataSyncAdapter {
|
|
|
104
104
|
streamExport?(entityType: string, cursor: string | null, config: SyncConfig): AsyncIterable<ExportBatch>
|
|
105
105
|
getInitialCursor?(entityType: string): Promise<string | null>
|
|
106
106
|
getMapping?(entityType: string): Promise<FieldMapping[]>
|
|
107
|
+
persistsSharedCursor?(entityType: string): boolean
|
|
107
108
|
validateConnection?(credentials: Record<string, unknown>): Promise<{ valid: boolean; message?: string }>
|
|
108
109
|
}
|
|
109
110
|
```
|
|
@@ -124,7 +125,9 @@ If the sync provider needs bootstrap credentials, mappings, locales, channels, o
|
|
|
124
125
|
|
|
125
126
|
`pending` → `running` → `completed` | `failed` | `cancelled`
|
|
126
127
|
|
|
127
|
-
- **Cursor persistence**: After each batch, cursor is saved
|
|
128
|
+
- **Cursor persistence**: After each batch, the cursor is saved on the run row and mirrored into the shared `SyncCursor` row
|
|
129
|
+
- **Shared cursor opt-out**: An adapter returning `persistsSharedCursor(entityType) === false` keeps that entity type's cursor on the run row only — use it for whole-table backfills whose cursor is one run's scan state, not a durable log position. Those entity types resolve an incremental start position from the most recent run (`resolveResumeCursor`) instead of the shared row, and from `null` when that run completed
|
|
130
|
+
- **Resetting an opt-out**: A reset flow that deletes the shared `SyncCursor` row MUST also call `syncRunService.resetResumePosition(integrationId, entityType, direction, scope)`. An opted-out entity type has no shared row to delete, so deleting only that leaves the resume position on the last interrupted run and the next incremental run re-imports just the tail of the walk it was reset against. The call is a no-op when nothing is interrupted, so make it unconditionally
|
|
128
131
|
- **Resume**: Retry reads the last successful cursor, resumes from there
|
|
129
132
|
- **Progress**: Linked to `ProgressJob` via `progressJobId` for `ProgressTopBar` display
|
|
130
133
|
- **Cancellation**: Via `progressService.isCancellationRequested()`
|
|
@@ -10,6 +10,7 @@ import type { SyncRunService } from '../lib/sync-run-service'
|
|
|
10
10
|
import { runSyncSchema } from '../data/validators'
|
|
11
11
|
import { startDataSyncRun } from '../lib/start-run'
|
|
12
12
|
import { getDataSyncAdapter } from '../lib/adapter-registry'
|
|
13
|
+
import { resolveStartCursor } from '../lib/start-cursor'
|
|
13
14
|
import {
|
|
14
15
|
runCrudMutationGuardAfterSuccess,
|
|
15
16
|
validateCrudMutationGuard,
|
|
@@ -109,7 +110,14 @@ export async function POST(req: Request) {
|
|
|
109
110
|
|
|
110
111
|
const cursor = parsed.data.fullSync
|
|
111
112
|
? null
|
|
112
|
-
: await
|
|
113
|
+
: await resolveStartCursor({
|
|
114
|
+
syncRunService,
|
|
115
|
+
adapter,
|
|
116
|
+
integrationId: parsed.data.integrationId,
|
|
117
|
+
entityType: parsed.data.entityType,
|
|
118
|
+
direction: parsed.data.direction,
|
|
119
|
+
scope,
|
|
120
|
+
})
|
|
113
121
|
|
|
114
122
|
const { run, progressJob } = await startDataSyncRun({
|
|
115
123
|
syncRunService,
|
|
@@ -8,6 +8,7 @@ import type { ProgressService } from '../../../../progress/lib/progressService'
|
|
|
8
8
|
import type { SyncRunService } from '../../../lib/sync-run-service'
|
|
9
9
|
import { retrySyncSchema } from '../../../data/validators'
|
|
10
10
|
import { startDataSyncRun } from '../../../lib/start-run'
|
|
11
|
+
import { resolveAdapterForIntegration, resolveStartCursor } from '../../../lib/start-cursor'
|
|
11
12
|
import {
|
|
12
13
|
runCrudMutationGuardAfterSuccess,
|
|
13
14
|
validateCrudMutationGuard,
|
|
@@ -89,7 +90,14 @@ export async function POST(req: Request, ctx: { params?: Promise<{ id?: string }
|
|
|
89
90
|
|
|
90
91
|
const cursor = parsedBody.data.fromBeginning
|
|
91
92
|
? null
|
|
92
|
-
: previous.cursor ?? await
|
|
93
|
+
: previous.cursor ?? await resolveStartCursor({
|
|
94
|
+
syncRunService,
|
|
95
|
+
adapter: resolveAdapterForIntegration(previous.integrationId),
|
|
96
|
+
integrationId: previous.integrationId,
|
|
97
|
+
entityType: previous.entityType,
|
|
98
|
+
direction: previous.direction,
|
|
99
|
+
scope,
|
|
100
|
+
})
|
|
93
101
|
|
|
94
102
|
const { run, progressJob } = await startDataSyncRun({
|
|
95
103
|
syncRunService,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getIntegration } from '@open-mercato/shared/modules/integrations/types'
|
|
1
2
|
import type { DataSyncAdapter } from './adapter'
|
|
2
3
|
|
|
3
4
|
const DATA_SYNC_ADAPTER_REGISTRY_KEY = Symbol.for('@open-mercato/data-sync/adapter-registry')
|
|
@@ -25,3 +26,17 @@ export function getDataSyncAdapter(providerKey: string): DataSyncAdapter | undef
|
|
|
25
26
|
export function getAllDataSyncAdapters(): DataSyncAdapter[] {
|
|
26
27
|
return Array.from(getAdapterRegistry().values())
|
|
27
28
|
}
|
|
29
|
+
|
|
30
|
+
export function resolveProviderKey(integrationId: string): string {
|
|
31
|
+
return getIntegration(integrationId)?.providerKey ?? integrationId
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The adapter serving an integration. Single source of truth on purpose: the
|
|
36
|
+
* engine uses it to decide whether to WRITE the shared cursor row and the start
|
|
37
|
+
* paths use it to decide whether to READ one, and those two decisions must agree
|
|
38
|
+
* for every integration or an opted-out entity type silently resumes wrong.
|
|
39
|
+
*/
|
|
40
|
+
export function resolveAdapterForIntegration(integrationId: string): DataSyncAdapter | null {
|
|
41
|
+
return getDataSyncAdapter(resolveProviderKey(integrationId)) ?? null
|
|
42
|
+
}
|
|
@@ -124,6 +124,24 @@ export interface DataSyncAdapter {
|
|
|
124
124
|
*/
|
|
125
125
|
streamImport?(input: StreamImportInput): AsyncIterable<ImportBatch>
|
|
126
126
|
streamExport?(input: StreamExportInput): AsyncIterable<ExportBatch>
|
|
127
|
+
/**
|
|
128
|
+
* Whether the engine mirrors this entity type's cursor into the shared
|
|
129
|
+
* `sync_cursors` row — one row per (integration, entityType, direction,
|
|
130
|
+
* scope), overwritten by every run that commits a batch.
|
|
131
|
+
*
|
|
132
|
+
* Default `true`. Return `false` for an entity type whose cursor is a single
|
|
133
|
+
* run's scan state rather than a durable position in a log. The run row keeps
|
|
134
|
+
* `initialCursor` + `cursor` either way, so a redelivered job still resumes
|
|
135
|
+
* exactly; what goes away is two concurrent or consecutive runs of the same
|
|
136
|
+
* entity type silently redefining each other's start position.
|
|
137
|
+
*
|
|
138
|
+
* The distinction is a blast radius, not a preference: losing a log position
|
|
139
|
+
* means re-draining the whole change queue, while losing a table walk's
|
|
140
|
+
* position means re-walking a table the adapter already re-walks idempotently.
|
|
141
|
+
* The predicate is per entity type because one adapter commonly serves both
|
|
142
|
+
* kinds — an incremental feed and a whole-table backfill.
|
|
143
|
+
*/
|
|
144
|
+
persistsSharedCursor?(entityType: string): boolean
|
|
127
145
|
getInitialCursor?(input: { entityType: string; scope: TenantScope }): Promise<string | null>
|
|
128
146
|
getMapping(input: { entityType: string; scope: TenantScope }): Promise<DataMapping>
|
|
129
147
|
validateConnection?(input: {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { DataSyncAdapter } from './adapter'
|
|
2
|
+
import type { SyncRunService } from './sync-run-service'
|
|
3
|
+
|
|
4
|
+
type SyncScope = {
|
|
5
|
+
organizationId: string
|
|
6
|
+
tenantId: string
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export { resolveAdapterForIntegration } from './adapter-registry'
|
|
10
|
+
|
|
11
|
+
export function persistsSharedCursor(adapter: DataSyncAdapter | null | undefined, entityType: string): boolean {
|
|
12
|
+
return adapter?.persistsSharedCursor?.(entityType) ?? true
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Start position for a non-full run. Entity types that mirror their cursor into
|
|
17
|
+
* the shared `sync_cursors` row read it from there. Entity types whose adapter
|
|
18
|
+
* opted out never write that row, so reading it would silently turn every
|
|
19
|
+
* incremental run into a full one — they resume from their own last run
|
|
20
|
+
* instead.
|
|
21
|
+
*/
|
|
22
|
+
export async function resolveStartCursor(params: {
|
|
23
|
+
syncRunService: SyncRunService
|
|
24
|
+
adapter?: DataSyncAdapter | null
|
|
25
|
+
integrationId: string
|
|
26
|
+
entityType: string
|
|
27
|
+
direction: 'import' | 'export'
|
|
28
|
+
scope: SyncScope
|
|
29
|
+
}): Promise<string | null> {
|
|
30
|
+
const { syncRunService, adapter, integrationId, entityType, direction, scope } = params
|
|
31
|
+
if (persistsSharedCursor(adapter, entityType)) {
|
|
32
|
+
return syncRunService.resolveCursor(integrationId, entityType, direction, scope)
|
|
33
|
+
}
|
|
34
|
+
return syncRunService.resolveResumeCursor(integrationId, entityType, direction, scope)
|
|
35
|
+
}
|
|
@@ -8,7 +8,7 @@ import { STALE_JOB_TIMEOUT_SECONDS } from '../../progress/lib/progressService'
|
|
|
8
8
|
import { refreshCoverageSnapshot } from '../../query_index/lib/coverage'
|
|
9
9
|
import { emitDataSyncEvent } from '../events'
|
|
10
10
|
import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'
|
|
11
|
-
import { getDataSyncAdapter } from './adapter-registry'
|
|
11
|
+
import { getDataSyncAdapter, resolveProviderKey } from './adapter-registry'
|
|
12
12
|
import type { SyncRunService } from './sync-run-service'
|
|
13
13
|
import { SyncRunOwnershipConflictError } from './sync-run-service'
|
|
14
14
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
@@ -30,10 +30,6 @@ type EngineDeps = {
|
|
|
30
30
|
progressService: ProgressService
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
function resolveProviderKey(integrationId: string): string {
|
|
34
|
-
return getIntegration(integrationId)?.providerKey ?? integrationId
|
|
35
|
-
}
|
|
36
|
-
|
|
37
33
|
function applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {
|
|
38
34
|
let createdCount = 0
|
|
39
35
|
let updatedCount = 0
|
|
@@ -486,6 +482,7 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
486
482
|
throw new Error(`No import adapter registered for provider ${providerKey}`)
|
|
487
483
|
}
|
|
488
484
|
const operationalTelemetry = adapter.operationalTelemetry === true
|
|
485
|
+
const persistSharedCursor = adapter.persistsSharedCursor?.(run.entityType) ?? true
|
|
489
486
|
|
|
490
487
|
const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)
|
|
491
488
|
if (!credentials) {
|
|
@@ -575,7 +572,7 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
575
572
|
},
|
|
576
573
|
batch.cursor,
|
|
577
574
|
scope,
|
|
578
|
-
committedBatches,
|
|
575
|
+
{ expectedBatchesCompleted: committedBatches, persistSharedCursor },
|
|
579
576
|
)
|
|
580
577
|
committedBatches += 1
|
|
581
578
|
|
|
@@ -650,6 +647,7 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
650
647
|
throw new Error(`No export adapter registered for provider ${providerKey}`)
|
|
651
648
|
}
|
|
652
649
|
const operationalTelemetry = adapter.operationalTelemetry === true
|
|
650
|
+
const persistSharedCursor = adapter.persistsSharedCursor?.(run.entityType) ?? true
|
|
653
651
|
|
|
654
652
|
const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)
|
|
655
653
|
if (!credentials) {
|
|
@@ -739,7 +737,7 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
739
737
|
},
|
|
740
738
|
batch.cursor,
|
|
741
739
|
scope,
|
|
742
|
-
committedBatches,
|
|
740
|
+
{ expectedBatchesCompleted: committedBatches, persistSharedCursor },
|
|
743
741
|
)
|
|
744
742
|
committedBatches += 1
|
|
745
743
|
await updateProgress(run.progressJobId, processedCount, null, scope)
|