@open-mercato/core 0.8.1-develop.7225.1.73190a4a95 → 0.8.1-develop.7228.1.b7ea09d2d3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/dist/modules/communication_channels/api/post/channels/[id]/import-history/route.js +5 -1
  3. package/dist/modules/communication_channels/api/post/channels/[id]/import-history/route.js.map +2 -2
  4. package/dist/modules/communication_channels/backend/profile/communication-channels/page.js +11 -7
  5. package/dist/modules/communication_channels/backend/profile/communication-channels/page.js.map +2 -2
  6. package/dist/modules/communication_channels/commands/queue-import-history.js +40 -7
  7. package/dist/modules/communication_channels/commands/queue-import-history.js.map +2 -2
  8. package/dist/modules/communication_channels/index.js +2 -1
  9. package/dist/modules/communication_channels/index.js.map +2 -2
  10. package/dist/modules/communication_channels/lib/import-history-limits.js +69 -0
  11. package/dist/modules/communication_channels/lib/import-history-limits.js.map +7 -0
  12. package/dist/modules/communication_channels/workers/channel-import-history.js +36 -2
  13. package/dist/modules/communication_channels/workers/channel-import-history.js.map +2 -2
  14. package/dist/modules/query_index/lib/engine.js +19 -1
  15. package/dist/modules/query_index/lib/engine.js.map +2 -2
  16. package/package.json +7 -7
  17. package/src/modules/communication_channels/api/post/channels/[id]/import-history/route.ts +6 -1
  18. package/src/modules/communication_channels/backend/profile/communication-channels/page.tsx +11 -7
  19. package/src/modules/communication_channels/commands/queue-import-history.ts +59 -11
  20. package/src/modules/communication_channels/i18n/de.json +2 -2
  21. package/src/modules/communication_channels/i18n/en.json +2 -2
  22. package/src/modules/communication_channels/i18n/es.json +2 -2
  23. package/src/modules/communication_channels/i18n/ko.json +2 -2
  24. package/src/modules/communication_channels/i18n/pl.json +2 -2
  25. package/src/modules/communication_channels/index.ts +1 -0
  26. package/src/modules/communication_channels/lib/adapter.ts +8 -2
  27. package/src/modules/communication_channels/lib/import-history-limits.ts +109 -0
  28. package/src/modules/communication_channels/workers/channel-import-history.ts +46 -5
  29. package/src/modules/query_index/lib/engine.ts +23 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@open-mercato/core",
3
- "version": "0.8.1-develop.7225.1.73190a4a95",
3
+ "version": "0.8.1-develop.7228.1.b7ea09d2d3",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -253,16 +253,16 @@
253
253
  "zod": "^4.4.3"
254
254
  },
255
255
  "peerDependencies": {
256
- "@open-mercato/ai-assistant": "0.8.1-develop.7225.1.73190a4a95",
257
- "@open-mercato/shared": "0.8.1-develop.7225.1.73190a4a95",
258
- "@open-mercato/ui": "0.8.1-develop.7225.1.73190a4a95",
256
+ "@open-mercato/ai-assistant": "0.8.1-develop.7228.1.b7ea09d2d3",
257
+ "@open-mercato/shared": "0.8.1-develop.7228.1.b7ea09d2d3",
258
+ "@open-mercato/ui": "0.8.1-develop.7228.1.b7ea09d2d3",
259
259
  "react": "^19.0.0",
260
260
  "react-dom": "^19.0.0"
261
261
  },
262
262
  "devDependencies": {
263
- "@open-mercato/ai-assistant": "0.8.1-develop.7225.1.73190a4a95",
264
- "@open-mercato/shared": "0.8.1-develop.7225.1.73190a4a95",
265
- "@open-mercato/ui": "0.8.1-develop.7225.1.73190a4a95",
263
+ "@open-mercato/ai-assistant": "0.8.1-develop.7228.1.b7ea09d2d3",
264
+ "@open-mercato/shared": "0.8.1-develop.7228.1.b7ea09d2d3",
265
+ "@open-mercato/ui": "0.8.1-develop.7228.1.b7ea09d2d3",
266
266
  "@testing-library/dom": "^10.4.1",
267
267
  "@testing-library/jest-dom": "^7.0.1",
268
268
  "@testing-library/react": "^16.3.3",
@@ -8,6 +8,7 @@ import type { CrudFormError } from '@open-mercato/ui/backend/utils/serverErrors'
8
8
  import { CommunicationChannel } from '../../../../../data/entities'
9
9
  import { ChannelAccessDeniedError, assertCanManageChannel } from '../../../../../lib/access-control'
10
10
  import {
11
+ getQueueImportHistorySchema,
11
12
  queueImportHistory,
12
13
  queueImportHistorySchema,
13
14
  } from '../../../../../commands/queue-import-history'
@@ -50,6 +51,10 @@ type RouteContext = {
50
51
 
51
52
  const bodySchema = queueImportHistorySchema.omit({ channelId: true })
52
53
 
54
+ function buildBodySchema() {
55
+ return getQueueImportHistorySchema().omit({ channelId: true })
56
+ }
57
+
53
58
  export async function POST(req: Request, context: RouteContext): Promise<Response> {
54
59
  const { id } = await context.params
55
60
  if (!z.string().uuid().safeParse(id).success) {
@@ -70,7 +75,7 @@ export async function POST(req: Request, context: RouteContext): Promise<Respons
70
75
  } catch {
71
76
  return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 })
72
77
  }
73
- const parsed = bodySchema.safeParse(rawBody)
78
+ const parsed = buildBodySchema().safeParse(rawBody)
74
79
  if (!parsed.success) {
75
80
  const first = parsed.error.issues[0]
76
81
  return NextResponse.json(
@@ -2,6 +2,7 @@
2
2
 
3
3
  import * as React from 'react'
4
4
  import { extensionPoints } from '@open-mercato/core/modules/communication_channels/extension-points'
5
+ import { getImportHistoryLimits } from '@open-mercato/core/modules/communication_channels/lib/import-history-limits'
5
6
  import type { LegacyColumnDef as ColumnDef } from '@tanstack/react-table/legacy'
6
7
  import { useRouter, useSearchParams } from 'next/navigation'
7
8
  import { Page, PageBody } from '@open-mercato/ui/backend/Page'
@@ -574,6 +575,7 @@ type ImportHistoryDialogProps = {
574
575
 
575
576
  function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialogProps): React.JSX.Element {
576
577
  const t = useT()
578
+ const importLimits = React.useMemo(() => getImportHistoryLimits(channel?.providerKey), [channel?.providerKey])
577
579
  const [sinceDays, setSinceDays] = React.useState('30')
578
580
  const [contactEmails, setContactEmails] = React.useState('')
579
581
  const [maxMessages, setMaxMessages] = React.useState('500')
@@ -599,16 +601,18 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
599
601
  const sinceNum = Number.parseInt(sinceDays, 10)
600
602
  const maxNum = Number.parseInt(maxMessages, 10)
601
603
  const errors: Record<string, string> = {}
602
- if (!Number.isFinite(sinceNum) || sinceNum < 1 || sinceNum > 365) {
604
+ if (!Number.isFinite(sinceNum) || sinceNum < 1 || sinceNum > importLimits.maxSinceDays) {
603
605
  errors.sinceDays = t(
604
606
  'communication_channels.profile.importHistory.errors.sinceDays',
605
- 'Choose a number between 1 and 365 days.',
607
+ 'Choose a number between 1 and {max} days.',
608
+ { max: importLimits.maxSinceDays },
606
609
  )
607
610
  }
608
- if (!Number.isFinite(maxNum) || maxNum < 1 || maxNum > 5000) {
611
+ if (!Number.isFinite(maxNum) || maxNum < 1 || maxNum > importLimits.maxMessages) {
609
612
  errors.maxMessages = t(
610
613
  'communication_channels.profile.importHistory.errors.maxMessages',
611
- 'Choose a number between 1 and 5000 messages.',
614
+ 'Choose a number between 1 and {max} messages.',
615
+ { max: importLimits.maxMessages },
612
616
  )
613
617
  }
614
618
  const parsedEmails = contactEmails
@@ -686,7 +690,7 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
686
690
  'success',
687
691
  )
688
692
  onQueued()
689
- }, [channel, sinceDays, maxMessages, contactEmails, submitting, t, onQueued, retryLastMutation, runMutation])
693
+ }, [channel, sinceDays, maxMessages, contactEmails, submitting, t, onQueued, retryLastMutation, runMutation, importLimits])
690
694
 
691
695
  const handleKeyDown = React.useCallback(
692
696
  (event: React.KeyboardEvent) => {
@@ -722,7 +726,7 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
722
726
  id="import-history-since"
723
727
  type="number"
724
728
  min={1}
725
- max={365}
729
+ max={importLimits.maxSinceDays}
726
730
  value={sinceDays}
727
731
  onChange={(e) => setSinceDays(e.target.value)}
728
732
  aria-invalid={Boolean(fieldErrors.sinceDays)}
@@ -770,7 +774,7 @@ function ImportHistoryDialog({ channel, onClose, onQueued }: ImportHistoryDialog
770
774
  id="import-history-max"
771
775
  type="number"
772
776
  min={1}
773
- max={5000}
777
+ max={importLimits.maxMessages}
774
778
  value={maxMessages}
775
779
  onChange={(e) => setMaxMessages(e.target.value)}
776
780
  aria-invalid={Boolean(fieldErrors.maxMessages)}
@@ -10,6 +10,11 @@ import type {
10
10
  import { CommunicationChannel } from '../data/entities'
11
11
  import { getChannelAdapterRegistry } from '../lib/adapter-registry-singleton'
12
12
  import { COMMUNICATION_CHANNELS_QUEUES, getCommunicationChannelsQueue } from '../lib/queue'
13
+ import {
14
+ IMPORT_HISTORY_MAX_CONTACT_EMAILS,
15
+ getImportHistoryLimits,
16
+ type ImportHistoryLimits,
17
+ } from '../lib/import-history-limits'
13
18
 
14
19
  /**
15
20
  * Spec B § Phase B6 — operator-triggered backlog import.
@@ -24,17 +29,40 @@ import { COMMUNICATION_CHANNELS_QUEUES, getCommunicationChannelsQueue } from '..
24
29
  * dropped guard is still safe.
25
30
  */
26
31
 
27
- export const queueImportHistorySchema = z.object({
28
- channelId: z.string().uuid(),
29
- sinceDays: z.number().int().min(1).max(365).default(30),
30
- contactEmails: z
31
- .array(z.string().email().max(255))
32
- .max(200)
33
- .optional(),
34
- maxMessages: z.number().int().min(1).max(5000).default(1000),
35
- })
32
+ export function createQueueImportHistorySchema(limits: ImportHistoryLimits = getImportHistoryLimits()) {
33
+ return z.object({
34
+ channelId: z.string().uuid(),
35
+ sinceDays: z.number().int().min(1).max(limits.maxSinceDays).default(limits.defaultSinceDays),
36
+ contactEmails: z
37
+ .array(z.string().email().max(255))
38
+ .max(IMPORT_HISTORY_MAX_CONTACT_EMAILS)
39
+ .optional(),
40
+ maxMessages: z.number().int().min(1).max(limits.maxMessages).default(limits.defaultMaxMessages),
41
+ })
42
+ }
43
+
44
+ export type QueueImportHistorySchema = ReturnType<typeof createQueueImportHistorySchema>
45
+
46
+ let cachedSchema: { key: string; schema: QueueImportHistorySchema } | null = null
47
+
48
+ /**
49
+ * The ceilings come from env, which a test may change after this module was
50
+ * imported, so the schema is rebuilt whenever the effective limits differ from
51
+ * the ones the cached instance was built with.
52
+ */
53
+ export function getQueueImportHistorySchema(): QueueImportHistorySchema {
54
+ const limits = getImportHistoryLimits()
55
+ const key = `${limits.maxSinceDays}:${limits.maxMessages}`
56
+ if (!cachedSchema || cachedSchema.key !== key) {
57
+ cachedSchema = { key, schema: createQueueImportHistorySchema(limits) }
58
+ }
59
+ return cachedSchema.schema
60
+ }
61
+
62
+ /** Retained for callers that need a statically typed schema instance. */
63
+ export const queueImportHistorySchema = getQueueImportHistorySchema()
36
64
 
37
- export type QueueImportHistoryInput = z.infer<typeof queueImportHistorySchema>
65
+ export type QueueImportHistoryInput = z.infer<QueueImportHistorySchema>
38
66
 
39
67
  export interface QueueImportHistoryScope {
40
68
  tenantId: string
@@ -63,7 +91,7 @@ export async function queueImportHistory(params: {
63
91
  scope: QueueImportHistoryScope
64
92
  input: QueueImportHistoryInput
65
93
  }): Promise<QueueImportHistoryResult> {
66
- const input = queueImportHistorySchema.parse(params.input)
94
+ const input = getQueueImportHistorySchema().parse(params.input)
67
95
  const { container, scope } = params
68
96
 
69
97
  const em = (container.resolve('em') as EntityManager).fork()
@@ -117,6 +145,26 @@ export async function queueImportHistory(params: {
117
145
  )
118
146
  }
119
147
 
148
+ // The schema above validated against the deployment-wide ceiling; some
149
+ // providers (IMAP) have a lower ceiling of their own (see
150
+ // `import-history-limits.ts`). Reject rather than silently truncate — the
151
+ // adapter itself has no way to report back "I clamped your request".
152
+ const providerLimits = getImportHistoryLimits(channel.providerKey)
153
+ if (input.sinceDays > providerLimits.maxSinceDays) {
154
+ throw createCrudFormError(
155
+ 'sinceDays exceeds the limit for this provider',
156
+ { sinceDays: `Provider "${channel.providerKey}" supports at most ${providerLimits.maxSinceDays} days` },
157
+ { status: 400 },
158
+ )
159
+ }
160
+ if (input.maxMessages > providerLimits.maxMessages) {
161
+ throw createCrudFormError(
162
+ 'maxMessages exceeds the limit for this provider',
163
+ { maxMessages: `Provider "${channel.providerKey}" supports at most ${providerLimits.maxMessages} messages` },
164
+ { status: 400 },
165
+ )
166
+ }
167
+
120
168
  const progressService = container.resolve('progressService') as ProgressService
121
169
  const progressContext: ProgressServiceContext = {
122
170
  tenantId: scope.tenantId,
@@ -110,8 +110,8 @@
110
110
  "communication_channels.profile.importHistory.cancel": "Abbrechen",
111
111
  "communication_channels.profile.importHistory.description": "Ältere Nachrichten abrufen, die dieser Kanal beim Verbinden noch nicht gesehen hat. Filter schränken die Suche serverseitig ein.",
112
112
  "communication_channels.profile.importHistory.errors.contactEmails": "Mindestens ein Eintrag ist keine gültige E-Mail-Adresse.",
113
- "communication_channels.profile.importHistory.errors.maxMessages": "Wählen Sie eine Zahl zwischen 1 und 5000 Nachrichten.",
114
- "communication_channels.profile.importHistory.errors.sinceDays": "Wählen Sie eine Zahl zwischen 1 und 365 Tagen.",
113
+ "communication_channels.profile.importHistory.errors.maxMessages": "Wählen Sie eine Zahl zwischen 1 und {max} Nachrichten.",
114
+ "communication_channels.profile.importHistory.errors.sinceDays": "Wählen Sie eine Zahl zwischen 1 und {max} Tagen.",
115
115
  "communication_channels.profile.importHistory.fields.contactEmails": "Nach Absender filtern (optional)",
116
116
  "communication_channels.profile.importHistory.fields.contactEmailsHint": "Leer lassen, um alle Absender im Zeitraum zu scannen.",
117
117
  "communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
@@ -110,8 +110,8 @@
110
110
  "communication_channels.profile.importHistory.cancel": "Cancel",
111
111
  "communication_channels.profile.importHistory.description": "Pull older messages this channel never observed at connect-time. Filters narrow the search server-side.",
112
112
  "communication_channels.profile.importHistory.errors.contactEmails": "One or more entries is not a valid email address.",
113
- "communication_channels.profile.importHistory.errors.maxMessages": "Choose a number between 1 and 5000 messages.",
114
- "communication_channels.profile.importHistory.errors.sinceDays": "Choose a number between 1 and 365 days.",
113
+ "communication_channels.profile.importHistory.errors.maxMessages": "Choose a number between 1 and {max} messages.",
114
+ "communication_channels.profile.importHistory.errors.sinceDays": "Choose a number between 1 and {max} days.",
115
115
  "communication_channels.profile.importHistory.fields.contactEmails": "Filter by sender (optional)",
116
116
  "communication_channels.profile.importHistory.fields.contactEmailsHint": "Leave empty to scan all senders in the window.",
117
117
  "communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
@@ -110,8 +110,8 @@
110
110
  "communication_channels.profile.importHistory.cancel": "Cancelar",
111
111
  "communication_channels.profile.importHistory.description": "Recupera mensajes anteriores que este canal no vio al conectarse. Los filtros restringen la búsqueda en el servidor.",
112
112
  "communication_channels.profile.importHistory.errors.contactEmails": "Una o más entradas no es una dirección de correo válida.",
113
- "communication_channels.profile.importHistory.errors.maxMessages": "Elige un número entre 1 y 5000 mensajes.",
114
- "communication_channels.profile.importHistory.errors.sinceDays": "Elige un número entre 1 y 365 días.",
113
+ "communication_channels.profile.importHistory.errors.maxMessages": "Elige un número entre 1 y {max} mensajes.",
114
+ "communication_channels.profile.importHistory.errors.sinceDays": "Elige un número entre 1 y {max} días.",
115
115
  "communication_channels.profile.importHistory.fields.contactEmails": "Filtrar por remitente (opcional)",
116
116
  "communication_channels.profile.importHistory.fields.contactEmailsHint": "Déjalo vacío para analizar todos los remitentes en el rango.",
117
117
  "communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
@@ -110,8 +110,8 @@
110
110
  "communication_channels.profile.importHistory.cancel": "취소",
111
111
  "communication_channels.profile.importHistory.description": "연결 시점에 확인되지 않았던 이전 메시지를 가져옵니다. 필터는 서버 측에서 검색 범위를 좁힙니다.",
112
112
  "communication_channels.profile.importHistory.errors.contactEmails": "하나 이상의 항목이 유효한 이메일 주소가 아닙니다.",
113
- "communication_channels.profile.importHistory.errors.maxMessages": "1에서 5000 사이의 메시지 수를 선택하세요.",
114
- "communication_channels.profile.importHistory.errors.sinceDays": "1에서 365일 사이의 일 수를 선택하세요.",
113
+ "communication_channels.profile.importHistory.errors.maxMessages": "1에서 {max} 사이의 메시지 수를 선택하세요.",
114
+ "communication_channels.profile.importHistory.errors.sinceDays": "1에서 {max}일 사이의 일 수를 선택하세요.",
115
115
  "communication_channels.profile.importHistory.fields.contactEmails": "발신자로 필터링 (선택 사항)",
116
116
  "communication_channels.profile.importHistory.fields.contactEmailsHint": "기간 내 모든 발신자를 검색하려면 비워두세요.",
117
117
  "communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
@@ -110,8 +110,8 @@
110
110
  "communication_channels.profile.importHistory.cancel": "Anuluj",
111
111
  "communication_channels.profile.importHistory.description": "Pobierz starsze wiadomości, których kanał nie widział w momencie połączenia. Filtry zawężają wyszukiwanie po stronie serwera.",
112
112
  "communication_channels.profile.importHistory.errors.contactEmails": "Co najmniej jeden wpis nie jest poprawnym adresem e-mail.",
113
- "communication_channels.profile.importHistory.errors.maxMessages": "Wybierz liczbę z zakresu od 1 do 5000 wiadomości.",
114
- "communication_channels.profile.importHistory.errors.sinceDays": "Wybierz liczbę z zakresu od 1 do 365 dni.",
113
+ "communication_channels.profile.importHistory.errors.maxMessages": "Wybierz liczbę z zakresu od 1 do {max} wiadomości.",
114
+ "communication_channels.profile.importHistory.errors.sinceDays": "Wybierz liczbę z zakresu od 1 do {max} dni.",
115
115
  "communication_channels.profile.importHistory.fields.contactEmails": "Filtruj po nadawcy (opcjonalnie)",
116
116
  "communication_channels.profile.importHistory.fields.contactEmailsHint": "Pozostaw puste, aby skanować wszystkich nadawców w zakresie.",
117
117
  "communication_channels.profile.importHistory.fields.contactEmailsPlaceholder": "alice@example.com, bob@example.com",
@@ -3,6 +3,7 @@ export const metadata = {
3
3
  title: 'Communication Channels',
4
4
  description:
5
5
  'Unified hub bridging external chat/email channels (Slack, WhatsApp, Email) to the Messages module. Adapter contract, channel-native payload storage, reactions, and per-channel threading.',
6
+ requires: ['progress'],
6
7
  }
7
8
 
8
9
  export default metadata
@@ -368,7 +368,10 @@ export interface ApplyPushNotificationInput {
368
368
  export interface ImportHistoryInput {
369
369
  credentials: Record<string, unknown>
370
370
  scope: TenantScope
371
- /** Look back at most this many days. Clamped 1..365 by the hub. */
371
+ /**
372
+ * Look back at most this many days. The hub accepts 1..`OM_IMPORT_HISTORY_MAX_SINCE_DAYS`
373
+ * (default ceiling 3650, i.e. ten years) and defaults to 30 when omitted.
374
+ */
372
375
  sinceDays: number
373
376
  /**
374
377
  * Optional sender-filter hint. Adapters SHOULD use it for server-side
@@ -376,7 +379,10 @@ export interface ImportHistoryInput {
376
379
  * import scans the entire `SINCE` window.
377
380
  */
378
381
  contactEmails?: string[]
379
- /** Total cap across all pages. Hub default 1000. Adapter MUST respect. */
382
+ /**
383
+ * Total cap across all pages. Hub default 1000, accepted up to
384
+ * `OM_IMPORT_HISTORY_MAX_MESSAGES` (default ceiling 50000). Adapter MUST respect.
385
+ */
380
386
  maxMessages?: number
381
387
  /** Opaque resumption cursor returned by the previous page. */
382
388
  cursor?: string
@@ -0,0 +1,109 @@
1
+ /**
2
+ * Single source of truth for the operator-triggered history-import bounds.
3
+ *
4
+ * Defaults are the values a request gets when it omits the field; ceilings are
5
+ * the largest value the hub accepts. Ceilings are deliberately wide enough for
6
+ * a multi-year backfill and can be raised or lowered per deployment through
7
+ * `OM_IMPORT_HISTORY_MAX_SINCE_DAYS` / `OM_IMPORT_HISTORY_MAX_MESSAGES`.
8
+ *
9
+ * Env is read at call time (not module load) so a deployment can change the
10
+ * ceiling without a rebuild, and so tests can exercise the override. In the
11
+ * browser bundle the variables are absent and the built-in ceilings apply —
12
+ * client-side validation is a UX affordance, the API remains the authority.
13
+ */
14
+
15
+ export const IMPORT_HISTORY_DEFAULT_SINCE_DAYS = 30
16
+ export const IMPORT_HISTORY_DEFAULT_MAX_MESSAGES = 1000
17
+
18
+ export const IMPORT_HISTORY_MAX_SINCE_DAYS_FALLBACK = 3650
19
+ export const IMPORT_HISTORY_MAX_MESSAGES_FALLBACK = 50000
20
+
21
+ export const IMPORT_HISTORY_MAX_CONTACT_EMAILS = 200
22
+
23
+ export const IMPORT_HISTORY_MIN_PAGE_BUDGET = 100
24
+ export const IMPORT_HISTORY_ABSOLUTE_MAX_PAGES = 10000
25
+
26
+ /**
27
+ * Consecutive pages that may return no messages while the adapter still claims
28
+ * `hasMore` and advances its cursor. Empty pages are legitimate: the Gmail
29
+ * adapter drops messages that returned 404/410 between listing and fetching,
30
+ * and server-side filters can eliminate a whole page. A stuck adapter is caught
31
+ * by the repeated-cursor check, so this counter only has to bound the run when
32
+ * the cursor keeps moving but nothing is ever returned.
33
+ */
34
+ export const IMPORT_HISTORY_MAX_EMPTY_PAGES = 10
35
+
36
+ export const IMPORT_HISTORY_MAX_SINCE_DAYS_ENV = 'OM_IMPORT_HISTORY_MAX_SINCE_DAYS'
37
+ export const IMPORT_HISTORY_MAX_MESSAGES_ENV = 'OM_IMPORT_HISTORY_MAX_MESSAGES'
38
+
39
+ export type ImportHistoryLimits = {
40
+ defaultSinceDays: number
41
+ defaultMaxMessages: number
42
+ maxSinceDays: number
43
+ maxMessages: number
44
+ }
45
+
46
+ function readPositiveIntEnv(name: string, fallback: number): number {
47
+ const raw = typeof process !== 'undefined' ? process.env?.[name] : undefined
48
+ const parsed = Number.parseInt(raw ?? '', 10)
49
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
50
+ }
51
+
52
+ export function getImportHistoryMaxSinceDays(): number {
53
+ return Math.max(
54
+ IMPORT_HISTORY_DEFAULT_SINCE_DAYS,
55
+ readPositiveIntEnv(IMPORT_HISTORY_MAX_SINCE_DAYS_ENV, IMPORT_HISTORY_MAX_SINCE_DAYS_FALLBACK),
56
+ )
57
+ }
58
+
59
+ export function getImportHistoryMaxMessages(): number {
60
+ return Math.max(
61
+ IMPORT_HISTORY_DEFAULT_MAX_MESSAGES,
62
+ readPositiveIntEnv(IMPORT_HISTORY_MAX_MESSAGES_ENV, IMPORT_HISTORY_MAX_MESSAGES_FALLBACK),
63
+ )
64
+ }
65
+
66
+ /**
67
+ * Per-provider ceilings, lower than the deployment-wide ceiling above.
68
+ *
69
+ * The IMAP adapter's cursor embeds the *entire* remaining UID list discovered
70
+ * by `SEARCH` (unlike Gmail's opaque `nextPageToken` cursor), so a wide ceiling
71
+ * there means a multi-megabyte cursor round-tripping through the worker on
72
+ * every page. Until that adapter gets a paging cursor of its own, its ceiling
73
+ * stays at the pre-widening values so the dialog, the hub's validation, and the
74
+ * adapter's own clamp (`packages/channel-imap/.../adapter.ts`) agree — a
75
+ * request above this is rejected rather than silently truncated.
76
+ */
77
+ const PROVIDER_MAX_SINCE_DAYS: Record<string, number> = {
78
+ imap: 365,
79
+ }
80
+ const PROVIDER_MAX_MESSAGES: Record<string, number> = {
81
+ imap: 5000,
82
+ }
83
+
84
+ export function getImportHistoryLimits(providerKey?: string): ImportHistoryLimits {
85
+ const maxSinceDays = getImportHistoryMaxSinceDays()
86
+ const maxMessages = getImportHistoryMaxMessages()
87
+ const providerMaxSinceDays = providerKey ? PROVIDER_MAX_SINCE_DAYS[providerKey] : undefined
88
+ const providerMaxMessages = providerKey ? PROVIDER_MAX_MESSAGES[providerKey] : undefined
89
+ return {
90
+ defaultSinceDays: IMPORT_HISTORY_DEFAULT_SINCE_DAYS,
91
+ defaultMaxMessages: IMPORT_HISTORY_DEFAULT_MAX_MESSAGES,
92
+ maxSinceDays: providerMaxSinceDays !== undefined ? Math.min(maxSinceDays, providerMaxSinceDays) : maxSinceDays,
93
+ maxMessages: providerMaxMessages !== undefined ? Math.min(maxMessages, providerMaxMessages) : maxMessages,
94
+ }
95
+ }
96
+
97
+ /**
98
+ * Page budget for the import drain loop, derived from the effective message cap
99
+ * so a large import is never truncated before it reaches that cap. It is only a
100
+ * backstop: an adapter that never makes progress is stopped much earlier by the
101
+ * consecutive-empty-page guard.
102
+ */
103
+ export function resolveImportHistoryPageBudget(maxMessages: number): number {
104
+ if (!Number.isFinite(maxMessages) || maxMessages <= 0) return IMPORT_HISTORY_MIN_PAGE_BUDGET
105
+ return Math.min(
106
+ IMPORT_HISTORY_ABSOLUTE_MAX_PAGES,
107
+ Math.max(IMPORT_HISTORY_MIN_PAGE_BUDGET, Math.ceil(maxMessages)),
108
+ )
109
+ }
@@ -10,6 +10,10 @@ import {
10
10
  import { COMMUNICATION_CHANNELS_QUEUES } from '../lib/queue'
11
11
  import { refreshCredentialsIfNeeded } from '../lib/credential-refresh'
12
12
  import { classifyOutboundError } from '../lib/error-classification'
13
+ import {
14
+ IMPORT_HISTORY_MAX_EMPTY_PAGES,
15
+ resolveImportHistoryPageBudget,
16
+ } from '../lib/import-history-limits'
13
17
  import type { ChannelAdapterRegistry } from '../lib/registry'
14
18
  import type {
15
19
  ChannelImportHistoryJobPayload,
@@ -159,16 +163,21 @@ export default async function handle(
159
163
  let totalCount = maxMessages
160
164
  let firstPage = true
161
165
 
162
- // Bounded loop guards against a misbehaving adapter (infinite hasMore).
163
- // 100 pages * HARD_CAP (default 200) = 20k messages — far above the 5k cap
164
- // enforced by the schema, so this only trips on adapter bugs.
165
- const MAX_PAGES = 100
166
- for (let pageIndex = 0; pageIndex < MAX_PAGES; pageIndex += 1) {
166
+ // Guards against an adapter that claims `hasMore` forever. An empty page is
167
+ // NOT a fault on its own — adapters legitimately drop every message on a
168
+ // page (Gmail 404/410 between listing and fetching, server-side filters) —
169
+ // so the hard stop is a cursor that stops advancing; the empty-page counter
170
+ // and the page budget are backstops.
171
+ const maxPages = resolveImportHistoryPageBudget(maxMessages)
172
+ let consecutiveEmptyPages = 0
173
+ let truncationReason: string | null = null
174
+ for (let pageIndex = 0; pageIndex < maxPages; pageIndex += 1) {
167
175
  if (await progressService.isCancellationRequested(progressJobId, scope.tenantId, scope.organizationId)) {
168
176
  await progressService.markCancelled(progressJobId, progressContext)
169
177
  return
170
178
  }
171
179
 
180
+ const requestedCursor = cursor
172
181
  const page = await adapter.importHistory!({
173
182
  credentials,
174
183
  scope: { tenantId: scope.tenantId, organizationId: scope.organizationId },
@@ -178,6 +187,21 @@ export default async function handle(
178
187
  cursor,
179
188
  })
180
189
 
190
+ if (page.hasMore && page.nextCursor !== undefined && page.nextCursor === requestedCursor) {
191
+ truncationReason = 'adapter repeated the same pagination cursor'
192
+ break
193
+ }
194
+
195
+ if (page.messages.length === 0 && page.hasMore) {
196
+ consecutiveEmptyPages += 1
197
+ if (consecutiveEmptyPages >= IMPORT_HISTORY_MAX_EMPTY_PAGES) {
198
+ truncationReason = `adapter returned no messages on ${consecutiveEmptyPages} consecutive pages`
199
+ break
200
+ }
201
+ } else {
202
+ consecutiveEmptyPages = 0
203
+ }
204
+
181
205
  if (firstPage && typeof page.totalCandidates === 'number') {
182
206
  totalCount = Math.min(maxMessages, page.totalCandidates)
183
207
  await progressService.updateProgress(
@@ -225,6 +249,23 @@ export default async function handle(
225
249
  if (processedCount >= maxMessages) break
226
250
  if (!page.hasMore || !page.nextCursor) break
227
251
  cursor = page.nextCursor
252
+ if (pageIndex === maxPages - 1) {
253
+ truncationReason = `page budget of ${maxPages} exhausted while the adapter still reported more messages`
254
+ }
255
+ }
256
+
257
+ if (truncationReason) {
258
+ // Surfaced as a failed job on purpose: the operator must see that the
259
+ // backlog is incomplete rather than a clean "imported N" summary.
260
+ logger.warn('history import stopped early', { channelId: channel.id, reason: truncationReason, processedCount })
261
+ await progressService.failJob(
262
+ progressJobId,
263
+ {
264
+ errorMessage: `History import stopped early after ${processedCount} messages: ${truncationReason}`,
265
+ },
266
+ progressContext,
267
+ )
268
+ return
228
269
  }
229
270
 
230
271
  await progressService.completeJob(
@@ -38,6 +38,7 @@ import { resolveListCountCap } from '@open-mercato/shared/lib/query/count-cap'
38
38
  import { mapWithConcurrency } from '@open-mercato/shared/lib/query/bounded-decrypt'
39
39
  import { createLogger } from '@open-mercato/shared/lib/logger'
40
40
  import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
41
+ import { createBoundedTtlMemo } from '@open-mercato/shared/lib/query/bounded-ttl-memo'
41
42
 
42
43
  const logger = createLogger('query_index').child({ component: 'engine' })
43
44
 
@@ -83,6 +84,24 @@ function markAutoReindexScheduled(key: string, debounceMs: number, now: number):
83
84
  return true
84
85
  }
85
86
 
87
+ /** Only `true` is cached: a cached `false` would keep routing to the fallback engine after a migration creates the table. */
88
+ const baseTableExistsCache = createBoundedTtlMemo<true>({
89
+ ttlEnv: 'OM_QUERY_INDEX_BASE_TABLE_EXISTS_CACHE_MS',
90
+ maxEntriesEnv: 'OM_QUERY_INDEX_BASE_TABLE_EXISTS_CACHE_MAX_ENTRIES',
91
+ defaultTtlMs: 3_600_000,
92
+ defaultMaxEntries: 1_000,
93
+ })
94
+
95
+ /** Test-only: the module-scoped memo would otherwise leak state across specs. */
96
+ export function clearBaseTableExistsCache(): void {
97
+ baseTableExistsCache.clear()
98
+ }
99
+
100
+ /** Test-only: entry count of the table-existence memo, for the cap regression test. */
101
+ export function baseTableExistsCacheSize(): number {
102
+ return baseTableExistsCache.size()
103
+ }
104
+
86
105
  function buildFilterableCustomFieldJoins(
87
106
  sources: QueryCustomFieldSource[] | undefined,
88
107
  ): Array<{
@@ -2162,13 +2181,16 @@ export class HybridQueryEngine implements QueryEngine {
2162
2181
  }
2163
2182
 
2164
2183
  private async tableExists(table: string): Promise<boolean> {
2184
+ if (baseTableExistsCache.get(table)) return true
2165
2185
  const db = this.getDb() as any
2166
2186
  const exists = await db
2167
2187
  .selectFrom('information_schema.tables')
2168
2188
  .select(sql<number>`1`.as('one'))
2169
2189
  .where('table_name', '=', table)
2170
2190
  .executeTakeFirst()
2171
- return !!exists
2191
+ const present = !!exists
2192
+ if (present) baseTableExistsCache.set(table, true)
2193
+ return present
2172
2194
  }
2173
2195
 
2174
2196
  private async resolveAvailableCustomFieldKeys(entityIds: string[], tenantId: string | null): Promise<string[]> {