@open-mercato/core 0.8.1-develop.7225.1.73190a4a95 → 0.8.1-develop.7228.1.b7ea09d2d3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/communication_channels/api/post/channels/[id]/import-history/route.js +5 -1
- package/dist/modules/communication_channels/api/post/channels/[id]/import-history/route.js.map +2 -2
- package/dist/modules/communication_channels/backend/profile/communication-channels/page.js +11 -7
- package/dist/modules/communication_channels/backend/profile/communication-channels/page.js.map +2 -2
- package/dist/modules/communication_channels/commands/queue-import-history.js +40 -7
- package/dist/modules/communication_channels/commands/queue-import-history.js.map +2 -2
- package/dist/modules/communication_channels/index.js +2 -1
- package/dist/modules/communication_channels/index.js.map +2 -2
- package/dist/modules/communication_channels/lib/import-history-limits.js +69 -0
- package/dist/modules/communication_channels/lib/import-history-limits.js.map +7 -0
- package/dist/modules/communication_channels/workers/channel-import-history.js +36 -2
- package/dist/modules/communication_channels/workers/channel-import-history.js.map +2 -2
- package/dist/modules/query_index/lib/engine.js +19 -1
- package/dist/modules/query_index/lib/engine.js.map +2 -2
- package/package.json +7 -7
- package/src/modules/communication_channels/api/post/channels/[id]/import-history/route.ts +6 -1
- package/src/modules/communication_channels/backend/profile/communication-channels/page.tsx +11 -7
- package/src/modules/communication_channels/commands/queue-import-history.ts +59 -11
- package/src/modules/communication_channels/i18n/de.json +2 -2
- package/src/modules/communication_channels/i18n/en.json +2 -2
- package/src/modules/communication_channels/i18n/es.json +2 -2
- package/src/modules/communication_channels/i18n/ko.json +2 -2
- package/src/modules/communication_channels/i18n/pl.json +2 -2
- package/src/modules/communication_channels/index.ts +1 -0
- package/src/modules/communication_channels/lib/adapter.ts +8 -2
- package/src/modules/communication_channels/lib/import-history-limits.ts +109 -0
- package/src/modules/communication_channels/workers/channel-import-history.ts +46 -5
- package/src/modules/query_index/lib/engine.ts +23 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/core",
|
|
3
|
-
"version": "0.8.1-develop.
|
|
3
|
+
"version": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -253,16 +253,16 @@
|
|
|
253
253
|
"zod": "^4.4.3"
|
|
254
254
|
},
|
|
255
255
|
"peerDependencies": {
|
|
256
|
-
"@open-mercato/ai-assistant": "0.8.1-develop.
|
|
257
|
-
"@open-mercato/shared": "0.8.1-develop.
|
|
258
|
-
"@open-mercato/ui": "0.8.1-develop.
|
|
256
|
+
"@open-mercato/ai-assistant": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
257
|
+
"@open-mercato/shared": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
258
|
+
"@open-mercato/ui": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
259
259
|
"react": "^19.0.0",
|
|
260
260
|
"react-dom": "^19.0.0"
|
|
261
261
|
},
|
|
262
262
|
"devDependencies": {
|
|
263
|
-
"@open-mercato/ai-assistant": "0.8.1-develop.
|
|
264
|
-
"@open-mercato/shared": "0.8.1-develop.
|
|
265
|
-
"@open-mercato/ui": "0.8.1-develop.
|
|
263
|
+
"@open-mercato/ai-assistant": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
264
|
+
"@open-mercato/shared": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
265
|
+
"@open-mercato/ui": "0.8.1-develop.7228.1.b7ea09d2d3",
|
|
266
266
|
"@testing-library/dom": "^10.4.1",
|
|
267
267
|
"@testing-library/jest-dom": "^7.0.1",
|
|
268
268
|
"@testing-library/react": "^16.3.3",
|
|
@@ -8,6 +8,7 @@ import type { CrudFormError } from '@open-mercato/ui/backend/utils/serverErrors'
|
|
|
8
8
|
import { CommunicationChannel } from '../../../../../data/entities'
|
|
9
9
|
import { ChannelAccessDeniedError, assertCanManageChannel } from '../../../../../lib/access-control'
|
|
10
10
|
import {
|
|
11
|
+
getQueueImportHistorySchema,
|
|
11
12
|
queueImportHistory,
|
|
12
13
|
queueImportHistorySchema,
|
|
13
14
|
} from '../../../../../commands/queue-import-history'
|
|
@@ -50,6 +51,10 @@ type RouteContext = {
|
|
|
50
51
|
|
|
51
52
|
const bodySchema = queueImportHistorySchema.omit({ channelId: true })
|
|
52
53
|
|
|
54
|
+
function buildBodySchema() {
|
|
55
|
+
return getQueueImportHistorySchema().omit({ channelId: true })
|
|
56
|
+
}
|
|
57
|
+
|
|
53
58
|
export async function POST(req: Request, context: RouteContext): Promise<Response> {
|
|
54
59
|
const { id } = await context.params
|
|
55
60
|
if (!z.string().uuid().safeParse(id).success) {
|
|
@@ -70,7 +75,7 @@ export async function POST(req: Request, context: RouteContext): Promise<Respons
|
|
|
70
75
|
} catch {
|
|
71
76
|
return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 })
|
|
72
77
|
}
|
|
73
|
-
const parsed =
|
|
78
|
+
const parsed = buildBodySchema().safeParse(rawBody)
|
|
74
79
|
if (!parsed.success) {
|
|
75
80
|
const first = parsed.error.issues[0]
|
|
76
81
|
return NextResponse.json(
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import * as React from 'react'
|
|
4
4
|
import { extensionPoints } from '@open-mercato/core/modules/communication_channels/extension-points'
|
|
5
|
+
import { getImportHistoryLimits } from '@open-mercato/core/modules/communication_channels/lib/import-history-limits'
|
|
5
6
|
import type { LegacyColumnDef as ColumnDef } from '@tanstack/react-table/legacy'
|
|
6
7
|
import { useRouter, useSearchParams } from 'next/navigation'
|
|
7
8
|
import { Page, PageBody } from '@open-mercato/ui/backend/Page'
|
|
@@ -574,6 +575,7 @@ type ImportHistoryDialogProps = {
|
|
|
574
575
|
|
|
575
576
|
function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialogProps): React.JSX.Element {
|
|
576
577
|
const t = useT()
|
|
578
|
+
const importLimits = React.useMemo(() => getImportHistoryLimits(channel?.providerKey), [channel?.providerKey])
|
|
577
579
|
const [sinceDays, setSinceDays] = React.useState('30')
|
|
578
580
|
const [contactEmails, setContactEmails] = React.useState('')
|
|
579
581
|
const [maxMessages, setMaxMessages] = React.useState('500')
|
|
@@ -599,16 +601,18 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
|
|
|
599
601
|
const sinceNum = Number.parseInt(sinceDays, 10)
|
|
600
602
|
const maxNum = Number.parseInt(maxMessages, 10)
|
|
601
603
|
const errors: Record<string, string> = {}
|
|
602
|
-
if (!Number.isFinite(sinceNum) || sinceNum < 1 || sinceNum >
|
|
604
|
+
if (!Number.isFinite(sinceNum) || sinceNum < 1 || sinceNum > importLimits.maxSinceDays) {
|
|
603
605
|
errors.sinceDays = t(
|
|
604
606
|
'communication_channels.profile.importHistory.errors.sinceDays',
|
|
605
|
-
'Choose a number between 1 and
|
|
607
|
+
'Choose a number between 1 and {max} days.',
|
|
608
|
+
{ max: importLimits.maxSinceDays },
|
|
606
609
|
)
|
|
607
610
|
}
|
|
608
|
-
if (!Number.isFinite(maxNum) || maxNum < 1 || maxNum >
|
|
611
|
+
if (!Number.isFinite(maxNum) || maxNum < 1 || maxNum > importLimits.maxMessages) {
|
|
609
612
|
errors.maxMessages = t(
|
|
610
613
|
'communication_channels.profile.importHistory.errors.maxMessages',
|
|
611
|
-
'Choose a number between 1 and
|
|
614
|
+
'Choose a number between 1 and {max} messages.',
|
|
615
|
+
{ max: importLimits.maxMessages },
|
|
612
616
|
)
|
|
613
617
|
}
|
|
614
618
|
const parsedEmails = contactEmails
|
|
@@ -686,7 +690,7 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
|
|
|
686
690
|
'success',
|
|
687
691
|
)
|
|
688
692
|
onQueued()
|
|
689
|
-
}, [channel, sinceDays, maxMessages, contactEmails, submitting, t, onQueued, retryLastMutation, runMutation])
|
|
693
|
+
}, [channel, sinceDays, maxMessages, contactEmails, submitting, t, onQueued, retryLastMutation, runMutation, importLimits])
|
|
690
694
|
|
|
691
695
|
const handleKeyDown = React.useCallback(
|
|
692
696
|
(event: React.KeyboardEvent) => {
|
|
@@ -722,7 +726,7 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
|
|
|
722
726
|
id="import-history-since"
|
|
723
727
|
type="number"
|
|
724
728
|
min={1}
|
|
725
|
-
max={
|
|
729
|
+
max={importLimits.maxSinceDays}
|
|
726
730
|
value={sinceDays}
|
|
727
731
|
onChange={(e) => setSinceDays(e.target.value)}
|
|
728
732
|
aria-invalid={Boolean(fieldErrors.sinceDays)}
|
|
@@ -770,7 +774,7 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
|
|
|
770
774
|
id="import-history-max"
|
|
771
775
|
type="number"
|
|
772
776
|
min={1}
|
|
773
|
-
max={
|
|
777
|
+
max={importLimits.maxMessages}
|
|
774
778
|
value={maxMessages}
|
|
775
779
|
onChange={(e) => setMaxMessages(e.target.value)}
|
|
776
780
|
aria-invalid={Boolean(fieldErrors.maxMessages)}
|
|
@@ -10,6 +10,11 @@ import type {
|
|
|
10
10
|
import { CommunicationChannel } from '../data/entities'
|
|
11
11
|
import { getChannelAdapterRegistry } from '../lib/adapter-registry-singleton'
|
|
12
12
|
import { COMMUNICATION_CHANNELS_QUEUES, getCommunicationChannelsQueue } from '../lib/queue'
|
|
13
|
+
import {
|
|
14
|
+
IMPORT_HISTORY_MAX_CONTACT_EMAILS,
|
|
15
|
+
getImportHistoryLimits,
|
|
16
|
+
type ImportHistoryLimits,
|
|
17
|
+
} from '../lib/import-history-limits'
|
|
13
18
|
|
|
14
19
|
/**
|
|
15
20
|
* Spec B § Phase B6 — operator-triggered backlog import.
|
|
@@ -24,17 +29,40 @@ import { COMMUNICATION_CHANNELS_QUEUES, getCommunicationChannelsQueue } from '..
|
|
|
24
29
|
* dropped guard is still safe.
|
|
25
30
|
*/
|
|
26
31
|
|
|
27
|
-
export
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
export function createQueueImportHistorySchema(limits: ImportHistoryLimits = getImportHistoryLimits()) {
|
|
33
|
+
return z.object({
|
|
34
|
+
channelId: z.string().uuid(),
|
|
35
|
+
sinceDays: z.number().int().min(1).max(limits.maxSinceDays).default(limits.defaultSinceDays),
|
|
36
|
+
contactEmails: z
|
|
37
|
+
.array(z.string().email().max(255))
|
|
38
|
+
.max(IMPORT_HISTORY_MAX_CONTACT_EMAILS)
|
|
39
|
+
.optional(),
|
|
40
|
+
maxMessages: z.number().int().min(1).max(limits.maxMessages).default(limits.defaultMaxMessages),
|
|
41
|
+
})
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type QueueImportHistorySchema = ReturnType<typeof createQueueImportHistorySchema>
|
|
45
|
+
|
|
46
|
+
let cachedSchema: { key: string; schema: QueueImportHistorySchema } | null = null
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The ceilings come from env, which a test may change after this module was
|
|
50
|
+
* imported, so the schema is rebuilt whenever the effective limits differ from
|
|
51
|
+
* the ones the cached instance was built with.
|
|
52
|
+
*/
|
|
53
|
+
export function getQueueImportHistorySchema(): QueueImportHistorySchema {
|
|
54
|
+
const limits = getImportHistoryLimits()
|
|
55
|
+
const key = `${limits.maxSinceDays}:${limits.maxMessages}`
|
|
56
|
+
if (!cachedSchema || cachedSchema.key !== key) {
|
|
57
|
+
cachedSchema = { key, schema: createQueueImportHistorySchema(limits) }
|
|
58
|
+
}
|
|
59
|
+
return cachedSchema.schema
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Retained for callers that need a statically typed schema instance. */
|
|
63
|
+
export const queueImportHistorySchema = getQueueImportHistorySchema()
|
|
36
64
|
|
|
37
|
-
export type QueueImportHistoryInput = z.infer<
|
|
65
|
+
export type QueueImportHistoryInput = z.infer<QueueImportHistorySchema>
|
|
38
66
|
|
|
39
67
|
export interface QueueImportHistoryScope {
|
|
40
68
|
tenantId: string
|
|
@@ -63,7 +91,7 @@ export async function queueImportHistory(params: {
|
|
|
63
91
|
scope: QueueImportHistoryScope
|
|
64
92
|
input: QueueImportHistoryInput
|
|
65
93
|
}): Promise<QueueImportHistoryResult> {
|
|
66
|
-
const input =
|
|
94
|
+
const input = getQueueImportHistorySchema().parse(params.input)
|
|
67
95
|
const { container, scope } = params
|
|
68
96
|
|
|
69
97
|
const em = (container.resolve('em') as EntityManager).fork()
|
|
@@ -117,6 +145,26 @@ export async function queueImportHistory(params: {
|
|
|
117
145
|
)
|
|
118
146
|
}
|
|
119
147
|
|
|
148
|
+
// The schema above validated against the deployment-wide ceiling; some
|
|
149
|
+
// providers (IMAP) have a lower ceiling of their own (see
|
|
150
|
+
// `import-history-limits.ts`). Reject rather than silently truncate — the
|
|
151
|
+
// adapter itself has no way to report back "I clamped your request".
|
|
152
|
+
const providerLimits = getImportHistoryLimits(channel.providerKey)
|
|
153
|
+
if (input.sinceDays > providerLimits.maxSinceDays) {
|
|
154
|
+
throw createCrudFormError(
|
|
155
|
+
'sinceDays exceeds the limit for this provider',
|
|
156
|
+
{ sinceDays: `Provider "${channel.providerKey}" supports at most ${providerLimits.maxSinceDays} days` },
|
|
157
|
+
{ status: 400 },
|
|
158
|
+
)
|
|
159
|
+
}
|
|
160
|
+
if (input.maxMessages > providerLimits.maxMessages) {
|
|
161
|
+
throw createCrudFormError(
|
|
162
|
+
'maxMessages exceeds the limit for this provider',
|
|
163
|
+
{ maxMessages: `Provider "${channel.providerKey}" supports at most ${providerLimits.maxMessages} messages` },
|
|
164
|
+
{ status: 400 },
|
|
165
|
+
)
|
|
166
|
+
}
|
|
167
|
+
|
|
120
168
|
const progressService = container.resolve('progressService') as ProgressService
|
|
121
169
|
const progressContext: ProgressServiceContext = {
|
|
122
170
|
tenantId: scope.tenantId,
|
|
@@ -110,8 +110,8 @@
|
|
|
110
110
|
"communication_channels.profile.importHistory.cancel": "Abbrechen",
|
|
111
111
|
"communication_channels.profile.importHistory.description": "Ältere Nachrichten abrufen, die dieser Kanal beim Verbinden noch nicht gesehen hat. Filter schränken die Suche serverseitig ein.",
|
|
112
112
|
"communication_channels.profile.importHistory.errors.contactEmails": "Mindestens ein Eintrag ist keine gültige E-Mail-Adresse.",
|
|
113
|
-
"communication_channels.profile.importHistory.errors.maxMessages": "Wählen Sie eine Zahl zwischen 1 und
|
|
114
|
-
"communication_channels.profile.importHistory.errors.sinceDays": "Wählen Sie eine Zahl zwischen 1 und
|
|
113
|
+
"communication_channels.profile.importHistory.errors.maxMessages": "Wählen Sie eine Zahl zwischen 1 und {max} Nachrichten.",
|
|
114
|
+
"communication_channels.profile.importHistory.errors.sinceDays": "Wählen Sie eine Zahl zwischen 1 und {max} Tagen.",
|
|
115
115
|
"communication_channels.profile.importHistory.fields.contactEmails": "Nach Absender filtern (optional)",
|
|
116
116
|
"communication_channels.profile.importHistory.fields.contactEmailsHint": "Leer lassen, um alle Absender im Zeitraum zu scannen.",
|
|
117
117
|
"communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
|
|
@@ -110,8 +110,8 @@
|
|
|
110
110
|
"communication_channels.profile.importHistory.cancel": "Cancel",
|
|
111
111
|
"communication_channels.profile.importHistory.description": "Pull older messages this channel never observed at connect-time. Filters narrow the search server-side.",
|
|
112
112
|
"communication_channels.profile.importHistory.errors.contactEmails": "One or more entries is not a valid email address.",
|
|
113
|
-
"communication_channels.profile.importHistory.errors.maxMessages": "Choose a number between 1 and
|
|
114
|
-
"communication_channels.profile.importHistory.errors.sinceDays": "Choose a number between 1 and
|
|
113
|
+
"communication_channels.profile.importHistory.errors.maxMessages": "Choose a number between 1 and {max} messages.",
|
|
114
|
+
"communication_channels.profile.importHistory.errors.sinceDays": "Choose a number between 1 and {max} days.",
|
|
115
115
|
"communication_channels.profile.importHistory.fields.contactEmails": "Filter by sender (optional)",
|
|
116
116
|
"communication_channels.profile.importHistory.fields.contactEmailsHint": "Leave empty to scan all senders in the window.",
|
|
117
117
|
"communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
|
|
@@ -110,8 +110,8 @@
|
|
|
110
110
|
"communication_channels.profile.importHistory.cancel": "Cancelar",
|
|
111
111
|
"communication_channels.profile.importHistory.description": "Recupera mensajes anteriores que este canal no vio al conectarse. Los filtros restringen la búsqueda en el servidor.",
|
|
112
112
|
"communication_channels.profile.importHistory.errors.contactEmails": "Una o más entradas no es una dirección de correo válida.",
|
|
113
|
-
"communication_channels.profile.importHistory.errors.maxMessages": "Elige un número entre 1 y
|
|
114
|
-
"communication_channels.profile.importHistory.errors.sinceDays": "Elige un número entre 1 y
|
|
113
|
+
"communication_channels.profile.importHistory.errors.maxMessages": "Elige un número entre 1 y {max} mensajes.",
|
|
114
|
+
"communication_channels.profile.importHistory.errors.sinceDays": "Elige un número entre 1 y {max} días.",
|
|
115
115
|
"communication_channels.profile.importHistory.fields.contactEmails": "Filtrar por remitente (opcional)",
|
|
116
116
|
"communication_channels.profile.importHistory.fields.contactEmailsHint": "Déjalo vacío para analizar todos los remitentes en el rango.",
|
|
117
117
|
"communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
|
|
@@ -110,8 +110,8 @@
|
|
|
110
110
|
"communication_channels.profile.importHistory.cancel": "취소",
|
|
111
111
|
"communication_channels.profile.importHistory.description": "연결 시점에 확인되지 않았던 이전 메시지를 가져옵니다. 필터는 서버 측에서 검색 범위를 좁힙니다.",
|
|
112
112
|
"communication_channels.profile.importHistory.errors.contactEmails": "하나 이상의 항목이 유효한 이메일 주소가 아닙니다.",
|
|
113
|
-
"communication_channels.profile.importHistory.errors.maxMessages": "1에서
|
|
114
|
-
"communication_channels.profile.importHistory.errors.sinceDays": "1에서
|
|
113
|
+
"communication_channels.profile.importHistory.errors.maxMessages": "1에서 {max} 사이의 메시지 수를 선택하세요.",
|
|
114
|
+
"communication_channels.profile.importHistory.errors.sinceDays": "1에서 {max}일 사이의 일 수를 선택하세요.",
|
|
115
115
|
"communication_channels.profile.importHistory.fields.contactEmails": "발신자로 필터링 (선택 사항)",
|
|
116
116
|
"communication_channels.profile.importHistory.fields.contactEmailsHint": "기간 내 모든 발신자를 검색하려면 비워두세요.",
|
|
117
117
|
"communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
|
|
@@ -110,8 +110,8 @@
|
|
|
110
110
|
"communication_channels.profile.importHistory.cancel": "Anuluj",
|
|
111
111
|
"communication_channels.profile.importHistory.description": "Pobierz starsze wiadomości, których kanał nie widział w momencie połączenia. Filtry zawężają wyszukiwanie po stronie serwera.",
|
|
112
112
|
"communication_channels.profile.importHistory.errors.contactEmails": "Co najmniej jeden wpis nie jest poprawnym adresem e-mail.",
|
|
113
|
-
"communication_channels.profile.importHistory.errors.maxMessages": "Wybierz liczbę z zakresu od 1 do
|
|
114
|
-
"communication_channels.profile.importHistory.errors.sinceDays": "Wybierz liczbę z zakresu od 1 do
|
|
113
|
+
"communication_channels.profile.importHistory.errors.maxMessages": "Wybierz liczbę z zakresu od 1 do {max} wiadomości.",
|
|
114
|
+
"communication_channels.profile.importHistory.errors.sinceDays": "Wybierz liczbę z zakresu od 1 do {max} dni.",
|
|
115
115
|
"communication_channels.profile.importHistory.fields.contactEmails": "Filtruj po nadawcy (opcjonalnie)",
|
|
116
116
|
"communication_channels.profile.importHistory.fields.contactEmailsHint": "Pozostaw puste, aby skanować wszystkich nadawców w zakresie.",
|
|
117
117
|
"communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
|
|
@@ -3,6 +3,7 @@ export const metadata = {
|
|
|
3
3
|
title: 'Communication Channels',
|
|
4
4
|
description:
|
|
5
5
|
'Unified hub bridging external chat/email channels (Slack, WhatsApp, Email) to the Messages module. Adapter contract, channel-native payload storage, reactions, and per-channel threading.',
|
|
6
|
+
requires: ['progress'],
|
|
6
7
|
}
|
|
7
8
|
|
|
8
9
|
export default metadata
|
|
@@ -368,7 +368,10 @@ export interface ApplyPushNotificationInput {
|
|
|
368
368
|
export interface ImportHistoryInput {
|
|
369
369
|
credentials: Record<string, unknown>
|
|
370
370
|
scope: TenantScope
|
|
371
|
-
/**
|
|
371
|
+
/**
|
|
372
|
+
* Look back at most this many days. The hub accepts 1..`OM_IMPORT_HISTORY_MAX_SINCE_DAYS`
|
|
373
|
+
* (default ceiling 3650, i.e. ten years) and defaults to 30 when omitted.
|
|
374
|
+
*/
|
|
372
375
|
sinceDays: number
|
|
373
376
|
/**
|
|
374
377
|
* Optional sender-filter hint. Adapters SHOULD use it for server-side
|
|
@@ -376,7 +379,10 @@ export interface ImportHistoryInput {
|
|
|
376
379
|
* import scans the entire `SINCE` window.
|
|
377
380
|
*/
|
|
378
381
|
contactEmails?: string[]
|
|
379
|
-
/**
|
|
382
|
+
/**
|
|
383
|
+
* Total cap across all pages. Hub default 1000, accepted up to
|
|
384
|
+
* `OM_IMPORT_HISTORY_MAX_MESSAGES` (default ceiling 50000). Adapter MUST respect.
|
|
385
|
+
*/
|
|
380
386
|
maxMessages?: number
|
|
381
387
|
/** Opaque resumption cursor returned by the previous page. */
|
|
382
388
|
cursor?: string
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Single source of truth for the operator-triggered history-import bounds.
|
|
3
|
+
*
|
|
4
|
+
* Defaults are the values a request gets when it omits the field; ceilings are
|
|
5
|
+
* the largest value the hub accepts. Ceilings are deliberately wide enough for
|
|
6
|
+
* a multi-year backfill and can be raised or lowered per deployment through
|
|
7
|
+
* `OM_IMPORT_HISTORY_MAX_SINCE_DAYS` / `OM_IMPORT_HISTORY_MAX_MESSAGES`.
|
|
8
|
+
*
|
|
9
|
+
* Env is read at call time (not module load) so a deployment can change the
|
|
10
|
+
* ceiling without a rebuild, and so tests can exercise the override. In the
|
|
11
|
+
* browser bundle the variables are absent and the built-in ceilings apply —
|
|
12
|
+
* client-side validation is a UX affordance, the API remains the authority.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export const IMPORT_HISTORY_DEFAULT_SINCE_DAYS = 30
|
|
16
|
+
export const IMPORT_HISTORY_DEFAULT_MAX_MESSAGES = 1000
|
|
17
|
+
|
|
18
|
+
export const IMPORT_HISTORY_MAX_SINCE_DAYS_FALLBACK = 3650
|
|
19
|
+
export const IMPORT_HISTORY_MAX_MESSAGES_FALLBACK = 50000
|
|
20
|
+
|
|
21
|
+
export const IMPORT_HISTORY_MAX_CONTACT_EMAILS = 200
|
|
22
|
+
|
|
23
|
+
export const IMPORT_HISTORY_MIN_PAGE_BUDGET = 100
|
|
24
|
+
export const IMPORT_HISTORY_ABSOLUTE_MAX_PAGES = 10000
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Consecutive pages that may return no messages while the adapter still claims
|
|
28
|
+
* `hasMore` and advances its cursor. Empty pages are legitimate: the Gmail
|
|
29
|
+
* adapter drops messages that returned 404/410 between listing and fetching,
|
|
30
|
+
* and server-side filters can eliminate a whole page. A stuck adapter is caught
|
|
31
|
+
* by the repeated-cursor check, so this counter only has to bound the run when
|
|
32
|
+
* the cursor keeps moving but nothing is ever returned.
|
|
33
|
+
*/
|
|
34
|
+
export const IMPORT_HISTORY_MAX_EMPTY_PAGES = 10
|
|
35
|
+
|
|
36
|
+
export const IMPORT_HISTORY_MAX_SINCE_DAYS_ENV = 'OM_IMPORT_HISTORY_MAX_SINCE_DAYS'
|
|
37
|
+
export const IMPORT_HISTORY_MAX_MESSAGES_ENV = 'OM_IMPORT_HISTORY_MAX_MESSAGES'
|
|
38
|
+
|
|
39
|
+
export type ImportHistoryLimits = {
|
|
40
|
+
defaultSinceDays: number
|
|
41
|
+
defaultMaxMessages: number
|
|
42
|
+
maxSinceDays: number
|
|
43
|
+
maxMessages: number
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function readPositiveIntEnv(name: string, fallback: number): number {
|
|
47
|
+
const raw = typeof process !== 'undefined' ? process.env?.[name] : undefined
|
|
48
|
+
const parsed = Number.parseInt(raw ?? '', 10)
|
|
49
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function getImportHistoryMaxSinceDays(): number {
|
|
53
|
+
return Math.max(
|
|
54
|
+
IMPORT_HISTORY_DEFAULT_SINCE_DAYS,
|
|
55
|
+
readPositiveIntEnv(IMPORT_HISTORY_MAX_SINCE_DAYS_ENV, IMPORT_HISTORY_MAX_SINCE_DAYS_FALLBACK),
|
|
56
|
+
)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function getImportHistoryMaxMessages(): number {
|
|
60
|
+
return Math.max(
|
|
61
|
+
IMPORT_HISTORY_DEFAULT_MAX_MESSAGES,
|
|
62
|
+
readPositiveIntEnv(IMPORT_HISTORY_MAX_MESSAGES_ENV, IMPORT_HISTORY_MAX_MESSAGES_FALLBACK),
|
|
63
|
+
)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Per-provider ceilings, lower than the deployment-wide ceiling above.
|
|
68
|
+
*
|
|
69
|
+
* The IMAP adapter's cursor embeds the *entire* remaining UID list discovered
|
|
70
|
+
* by `SEARCH` (unlike Gmail's opaque `nextPageToken` cursor), so a wide ceiling
|
|
71
|
+
* there means a multi-megabyte cursor round-tripping through the worker on
|
|
72
|
+
* every page. Until that adapter gets a paging cursor of its own, its ceiling
|
|
73
|
+
* stays at the pre-widening values so the dialog, the hub's validation, and the
|
|
74
|
+
* adapter's own clamp (`packages/channel-imap/.../adapter.ts`) agree — a
|
|
75
|
+
* request above this is rejected rather than silently truncated.
|
|
76
|
+
*/
|
|
77
|
+
const PROVIDER_MAX_SINCE_DAYS: Record<string, number> = {
|
|
78
|
+
imap: 365,
|
|
79
|
+
}
|
|
80
|
+
const PROVIDER_MAX_MESSAGES: Record<string, number> = {
|
|
81
|
+
imap: 5000,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function getImportHistoryLimits(providerKey?: string): ImportHistoryLimits {
|
|
85
|
+
const maxSinceDays = getImportHistoryMaxSinceDays()
|
|
86
|
+
const maxMessages = getImportHistoryMaxMessages()
|
|
87
|
+
const providerMaxSinceDays = providerKey ? PROVIDER_MAX_SINCE_DAYS[providerKey] : undefined
|
|
88
|
+
const providerMaxMessages = providerKey ? PROVIDER_MAX_MESSAGES[providerKey] : undefined
|
|
89
|
+
return {
|
|
90
|
+
defaultSinceDays: IMPORT_HISTORY_DEFAULT_SINCE_DAYS,
|
|
91
|
+
defaultMaxMessages: IMPORT_HISTORY_DEFAULT_MAX_MESSAGES,
|
|
92
|
+
maxSinceDays: providerMaxSinceDays !== undefined ? Math.min(maxSinceDays, providerMaxSinceDays) : maxSinceDays,
|
|
93
|
+
maxMessages: providerMaxMessages !== undefined ? Math.min(maxMessages, providerMaxMessages) : maxMessages,
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Page budget for the import drain loop, derived from the effective message cap
|
|
99
|
+
* so a large import is never truncated before it reaches that cap. It is only a
|
|
100
|
+
* backstop: an adapter that never makes progress is stopped much earlier by the
|
|
101
|
+
* consecutive-empty-page guard.
|
|
102
|
+
*/
|
|
103
|
+
export function resolveImportHistoryPageBudget(maxMessages: number): number {
|
|
104
|
+
if (!Number.isFinite(maxMessages) || maxMessages <= 0) return IMPORT_HISTORY_MIN_PAGE_BUDGET
|
|
105
|
+
return Math.min(
|
|
106
|
+
IMPORT_HISTORY_ABSOLUTE_MAX_PAGES,
|
|
107
|
+
Math.max(IMPORT_HISTORY_MIN_PAGE_BUDGET, Math.ceil(maxMessages)),
|
|
108
|
+
)
|
|
109
|
+
}
|
|
@@ -10,6 +10,10 @@ import {
|
|
|
10
10
|
import { COMMUNICATION_CHANNELS_QUEUES } from '../lib/queue'
|
|
11
11
|
import { refreshCredentialsIfNeeded } from '../lib/credential-refresh'
|
|
12
12
|
import { classifyOutboundError } from '../lib/error-classification'
|
|
13
|
+
import {
|
|
14
|
+
IMPORT_HISTORY_MAX_EMPTY_PAGES,
|
|
15
|
+
resolveImportHistoryPageBudget,
|
|
16
|
+
} from '../lib/import-history-limits'
|
|
13
17
|
import type { ChannelAdapterRegistry } from '../lib/registry'
|
|
14
18
|
import type {
|
|
15
19
|
ChannelImportHistoryJobPayload,
|
|
@@ -159,16 +163,21 @@ export default async function handle(
|
|
|
159
163
|
let totalCount = maxMessages
|
|
160
164
|
let firstPage = true
|
|
161
165
|
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
|
|
166
|
+
// Guards against an adapter that claims `hasMore` forever. An empty page is
|
|
167
|
+
// NOT a fault on its own — adapters legitimately drop every message on a
|
|
168
|
+
// page (Gmail 404/410 between listing and fetching, server-side filters) —
|
|
169
|
+
// so the hard stop is a cursor that stops advancing; the empty-page counter
|
|
170
|
+
// and the page budget are backstops.
|
|
171
|
+
const maxPages = resolveImportHistoryPageBudget(maxMessages)
|
|
172
|
+
let consecutiveEmptyPages = 0
|
|
173
|
+
let truncationReason: string | null = null
|
|
174
|
+
for (let pageIndex = 0; pageIndex < maxPages; pageIndex += 1) {
|
|
167
175
|
if (await progressService.isCancellationRequested(progressJobId, scope.tenantId, scope.organizationId)) {
|
|
168
176
|
await progressService.markCancelled(progressJobId, progressContext)
|
|
169
177
|
return
|
|
170
178
|
}
|
|
171
179
|
|
|
180
|
+
const requestedCursor = cursor
|
|
172
181
|
const page = await adapter.importHistory!({
|
|
173
182
|
credentials,
|
|
174
183
|
scope: { tenantId: scope.tenantId, organizationId: scope.organizationId },
|
|
@@ -178,6 +187,21 @@ export default async function handle(
|
|
|
178
187
|
cursor,
|
|
179
188
|
})
|
|
180
189
|
|
|
190
|
+
if (page.hasMore && page.nextCursor !== undefined && page.nextCursor === requestedCursor) {
|
|
191
|
+
truncationReason = 'adapter repeated the same pagination cursor'
|
|
192
|
+
break
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (page.messages.length === 0 && page.hasMore) {
|
|
196
|
+
consecutiveEmptyPages += 1
|
|
197
|
+
if (consecutiveEmptyPages >= IMPORT_HISTORY_MAX_EMPTY_PAGES) {
|
|
198
|
+
truncationReason = `adapter returned no messages on ${consecutiveEmptyPages} consecutive pages`
|
|
199
|
+
break
|
|
200
|
+
}
|
|
201
|
+
} else {
|
|
202
|
+
consecutiveEmptyPages = 0
|
|
203
|
+
}
|
|
204
|
+
|
|
181
205
|
if (firstPage && typeof page.totalCandidates === 'number') {
|
|
182
206
|
totalCount = Math.min(maxMessages, page.totalCandidates)
|
|
183
207
|
await progressService.updateProgress(
|
|
@@ -225,6 +249,23 @@ export default async function handle(
|
|
|
225
249
|
if (processedCount >= maxMessages) break
|
|
226
250
|
if (!page.hasMore || !page.nextCursor) break
|
|
227
251
|
cursor = page.nextCursor
|
|
252
|
+
if (pageIndex === maxPages - 1) {
|
|
253
|
+
truncationReason = `page budget of ${maxPages} exhausted while the adapter still reported more messages`
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
if (truncationReason) {
|
|
258
|
+
// Surfaced as a failed job on purpose: the operator must see that the
|
|
259
|
+
// backlog is incomplete rather than a clean "imported N" summary.
|
|
260
|
+
logger.warn('history import stopped early', { channelId: channel.id, reason: truncationReason, processedCount })
|
|
261
|
+
await progressService.failJob(
|
|
262
|
+
progressJobId,
|
|
263
|
+
{
|
|
264
|
+
errorMessage: `History import stopped early after ${processedCount} messages: ${truncationReason}`,
|
|
265
|
+
},
|
|
266
|
+
progressContext,
|
|
267
|
+
)
|
|
268
|
+
return
|
|
228
269
|
}
|
|
229
270
|
|
|
230
271
|
await progressService.completeJob(
|
|
@@ -38,6 +38,7 @@ import { resolveListCountCap } from '@open-mercato/shared/lib/query/count-cap'
|
|
|
38
38
|
import { mapWithConcurrency } from '@open-mercato/shared/lib/query/bounded-decrypt'
|
|
39
39
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
40
40
|
import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
|
|
41
|
+
import { createBoundedTtlMemo } from '@open-mercato/shared/lib/query/bounded-ttl-memo'
|
|
41
42
|
|
|
42
43
|
const logger = createLogger('query_index').child({ component: 'engine' })
|
|
43
44
|
|
|
@@ -83,6 +84,24 @@ function markAutoReindexScheduled(key: string, debounceMs: number, now: number):
|
|
|
83
84
|
return true
|
|
84
85
|
}
|
|
85
86
|
|
|
87
|
+
/** Only `true` is cached: a cached `false` would keep routing to the fallback engine after a migration creates the table. */
|
|
88
|
+
const baseTableExistsCache = createBoundedTtlMemo<true>({
|
|
89
|
+
ttlEnv: 'OM_QUERY_INDEX_BASE_TABLE_EXISTS_CACHE_MS',
|
|
90
|
+
maxEntriesEnv: 'OM_QUERY_INDEX_BASE_TABLE_EXISTS_CACHE_MAX_ENTRIES',
|
|
91
|
+
defaultTtlMs: 3_600_000,
|
|
92
|
+
defaultMaxEntries: 1_000,
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
/** Test-only: the module-scoped memo would otherwise leak state across specs. */
|
|
96
|
+
export function clearBaseTableExistsCache(): void {
|
|
97
|
+
baseTableExistsCache.clear()
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Test-only: entry count of the table-existence memo, for the cap regression test. */
|
|
101
|
+
export function baseTableExistsCacheSize(): number {
|
|
102
|
+
return baseTableExistsCache.size()
|
|
103
|
+
}
|
|
104
|
+
|
|
86
105
|
function buildFilterableCustomFieldJoins(
|
|
87
106
|
sources: QueryCustomFieldSource[] | undefined,
|
|
88
107
|
): Array<{
|
|
@@ -2162,13 +2181,16 @@ export class HybridQueryEngine implements QueryEngine {
|
|
|
2162
2181
|
}
|
|
2163
2182
|
|
|
2164
2183
|
private async tableExists(table: string): Promise<boolean> {
|
|
2184
|
+
if (baseTableExistsCache.get(table)) return true
|
|
2165
2185
|
const db = this.getDb() as any
|
|
2166
2186
|
const exists = await db
|
|
2167
2187
|
.selectFrom('information_schema.tables')
|
|
2168
2188
|
.select(sql<number>`1`.as('one'))
|
|
2169
2189
|
.where('table_name', '=', table)
|
|
2170
2190
|
.executeTakeFirst()
|
|
2171
|
-
|
|
2191
|
+
const present = !!exists
|
|
2192
|
+
if (present) baseTableExistsCache.set(table, true)
|
|
2193
|
+
return present
|
|
2172
2194
|
}
|
|
2173
2195
|
|
|
2174
2196
|
private async resolveAvailableCustomFieldKeys(entityIds: string[], tenantId: string | null): Promise<string[]> {
|