@open-mercato/core 0.9.1-develop.7459.1.99e3accd30 → 0.9.1-develop.7462.1.94c09f158f

package/src/modules/attachments/lib/ocrLimits.ts

@@ -0,0 +1,62 @@
+import { readEndpointRateLimitConfig } from '@open-mercato/shared/lib/ratelimit/config'
+import type { RateLimitConfig } from '@open-mercato/shared/lib/ratelimit/types'
+
+const DEFAULT_MAX_OCR_PAGES = 50
+const DEFAULT_OCR_PAGE_TIMEOUT_MS = 60_000
+const DEFAULT_OCR_MAX_OUTPUT_TOKENS = 4_096
+const DEFAULT_OCR_MAX_CONCURRENCY = 2
+const DEFAULT_OCR_MAX_WAIT_QUEUE = 50
+
+function parsePositiveInt(raw: string | undefined, fallback: number): number {
+  if (raw === undefined || raw.trim() === '') return fallback
+  const parsed = Number(raw)
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback
+  return Math.floor(parsed)
+}
+
+/** Cap PDF pages processed for OCR / text extraction (env: OM_ATTACHMENT_OCR_MAX_PAGES). */
+export function resolveMaxOcrPages(): number {
+  return parsePositiveInt(process.env.OM_ATTACHMENT_OCR_MAX_PAGES, DEFAULT_MAX_OCR_PAGES)
+}
+
+/** Per-page LLM OCR call timeout in ms (env: OM_ATTACHMENT_OCR_PAGE_TIMEOUT_MS). */
+export function resolveOcrPageTimeoutMs(): number {
+  return parsePositiveInt(process.env.OM_ATTACHMENT_OCR_PAGE_TIMEOUT_MS, DEFAULT_OCR_PAGE_TIMEOUT_MS)
+}
+
+/** Max output tokens for a single OCR generateText call (env: OM_ATTACHMENT_OCR_MAX_OUTPUT_TOKENS). */
+export function resolveOcrMaxOutputTokens(): number {
+  return parsePositiveInt(process.env.OM_ATTACHMENT_OCR_MAX_OUTPUT_TOKENS, DEFAULT_OCR_MAX_OUTPUT_TOKENS)
+}
+
+/** Max concurrent in-process OCR jobs (env: OM_ATTACHMENT_OCR_MAX_CONCURRENCY). */
+export function resolveOcrMaxConcurrency(): number {
+  return parsePositiveInt(process.env.OM_ATTACHMENT_OCR_MAX_CONCURRENCY, DEFAULT_OCR_MAX_CONCURRENCY)
+}
+
+/** Max waiters blocked on the in-process OCR concurrency slot (env: OM_ATTACHMENT_OCR_MAX_WAIT_QUEUE). */
+export function resolveOcrMaxWaitQueue(): number {
+  return parsePositiveInt(process.env.OM_ATTACHMENT_OCR_MAX_WAIT_QUEUE, DEFAULT_OCR_MAX_WAIT_QUEUE)
+}
+
+/** How many PDF pages to iterate given document length and the configured cap. */
+export function resolvePdfPageIterationLimit(numPages: number, maxPages: number = resolveMaxOcrPages()): number {
+  if (!Number.isFinite(numPages) || numPages <= 0) return 0
+  if (!Number.isFinite(maxPages) || maxPages <= 0) return 0
+  return Math.min(Math.floor(numPages), Math.floor(maxPages))
+}
+
+/**
+ * Per-tenant/user upload throttle for POST /api/attachments.
+ * Env: RATE_LIMIT_ATTACHMENTS_UPLOAD_{POINTS,DURATION,BLOCK_DURATION}.
+ */
+export function resolveAttachmentsUploadRateLimitConfig(): RateLimitConfig {
+  // Default sits above typical multi-file library batches (dialog has no hard
+  // file cap). Client retry/partial-progress UX for rejected mid-batch files is
+  // tracked in https://github.com/open-mercato/open-mercato/issues/6397.
+  return readEndpointRateLimitConfig('ATTACHMENTS_UPLOAD', {
+    points: 100,
+    duration: 60,
+    keyPrefix: 'attachments_upload',
+  })
+}

package/src/modules/attachments/lib/ocrQueue.ts

@@ -3,6 +3,8 @@ import { Attachment, AttachmentPartition } from '../data/entities'
 import { OcrService } from './ocrService'
 import type { StorageDriver } from './drivers/types'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveOcrMaxConcurrency, resolveOcrMaxWaitQueue } from './ocrLimits'
+import { extractAttachmentContent } from './textExtraction'
 
 const logger = createLogger('attachments').child({ component: 'ocr' })
 
@@ -15,6 +17,45 @@ export type OcrRequestedEvent = {
   tenantId: string | null
 }
 
+/** Outcome of `requestOcrProcessing` — background LLM OCR vs inline text fallback. */
+export type OcrProcessingDispatch = 'queued' | 'inline_fallback'
+
+let activeOcrJobs = 0
+const ocrWaitQueue: Array<() => void> = []
+// Jobs scheduled with setImmediate that have not yet entered a slot or the wait queue.
+// Counted toward the wait-queue cap so a burst in one event-loop turn cannot pass the check together.
+let pendingOcrJobs = 0
+
+/** Test-only: reset in-process OCR concurrency bookkeeping. */
+export function resetOcrConcurrencyStateForTests(): void {
+  activeOcrJobs = 0
+  ocrWaitQueue.length = 0
+  pendingOcrJobs = 0
+}
+
+/** Test-only: inspect in-process OCR concurrency counters. */
+export function getOcrConcurrencyStateForTests(): { active: number; waiting: number } {
+  return { active: activeOcrJobs, waiting: ocrWaitQueue.length }
+}
+
+export async function withOcrConcurrencySlot<T>(run: () => Promise<T>): Promise<T> {
+  const maxConcurrency = resolveOcrMaxConcurrency()
+  // Re-check after wake: multiple waiters can resume when one slot frees (barging).
+  while (activeOcrJobs >= maxConcurrency) {
+    await new Promise<void>((resolve) => {
+      ocrWaitQueue.push(resolve)
+    })
+  }
+  activeOcrJobs += 1
+  try {
+    return await run()
+  } finally {
+    activeOcrJobs -= 1
+    const next = ocrWaitQueue.shift()
+    if (next) next()
+  }
+}
+
 export async function processAttachmentOcr(
   em: EntityManager,
   payload: OcrRequestedEvent,
@@ -73,12 +114,49 @@ export async function processAttachmentOcr(
   }
 }
 
+/**
+ * When the in-process OCR wait queue is full, extract plain text/PDF/DOCX content
+ * inline (no LLM) so `attachment.content` is not left NULL forever.
+ */
+async function persistInlineTextExtractionFallback(
+  em: EntityManager,
+  payload: OcrRequestedEvent,
+  driver: StorageDriver,
+): Promise<void> {
+  const { attachmentId, storagePath, mimeType, partitionCode } = payload
+  const { filePath, cleanup } = await driver.toLocalPath(partitionCode, storagePath)
+  try {
+    const content = await extractAttachmentContent({ filePath, mimeType })
+    if (!content) {
+      logger.info('OCR wait queue full; inline extraction produced no content', { attachmentId })
+      return
+    }
+    const row = await em.findOne(Attachment, { id: attachmentId })
+    if (!row) {
+      logger.error('Attachment not found during OCR overflow fallback', { attachmentId })
+      return
+    }
+    row.content = content
+    await em.persist(row).flush()
+    logger.info('OCR wait queue full; stored inline text extraction', {
+      attachmentId,
+      contentLength: content.length,
+    })
+  } catch (error) {
+    logger.error('OCR wait queue overflow inline extraction failed', { attachmentId, err: error })
+  } finally {
+    await cleanup().catch((cleanupError) => {
+      logger.warn('Temp file cleanup failed after OCR overflow fallback', { err: cleanupError })
+    })
+  }
+}
+
 export async function requestOcrProcessing(
   em: EntityManager,
   attachment: Attachment,
   driver: StorageDriver,
   storagePath: string,
-): Promise<void> {
+): Promise<OcrProcessingDispatch> {
   const payload: OcrRequestedEvent = {
     attachmentId: attachment.id,
     storagePath,
@@ -97,9 +175,24 @@ export async function requestOcrProcessing(
 
   const workerEm = em.fork()
 
+  const maxWaitQueue = resolveOcrMaxWaitQueue()
+  if (ocrWaitQueue.length + pendingOcrJobs >= maxWaitQueue) {
+    logger.warn('OCR wait queue full; falling back to inline text extraction', {
+      attachmentId: attachment.id,
+      waiting: ocrWaitQueue.length,
+      pending: pendingOcrJobs,
+      maxWaitQueue,
+    })
+    await persistInlineTextExtractionFallback(workerEm, payload, driver)
+    return 'inline_fallback'
+  }
+
+  pendingOcrJobs += 1
   setImmediate(() => {
-    processAttachmentOcr(workerEm, payload, driver).catch((error) => {
+    pendingOcrJobs -= 1
+    withOcrConcurrencySlot(() => processAttachmentOcr(workerEm, payload, driver)).catch((error) => {
       logger.error('Background processing error', { err: error })
     })
   })
+  return 'queued'
 }

package/src/modules/attachments/lib/ocrService.ts

@@ -3,6 +3,7 @@ import { createOpenAI } from '@ai-sdk/openai'
 import fs from 'fs/promises'
 import path from 'path'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveOcrMaxOutputTokens, resolveOcrPageTimeoutMs } from './ocrLimits'
 
 const logger = createLogger('attachments').child({ component: 'ocr-service' })
 
@@ -102,6 +103,9 @@ export class OcrService {
     try {
       const result = await generateText({
         model: client(model),
+        abortSignal: AbortSignal.timeout(resolveOcrPageTimeoutMs()),
+        maxRetries: 0,
+        maxOutputTokens: resolveOcrMaxOutputTokens(),
         messages: [
           {
             role: 'user',

package/src/modules/attachments/lib/pdfProcessing.ts

@@ -1,11 +1,14 @@
 import fs from 'fs/promises'
 import path from 'path'
 import { createRequire } from 'module'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveMaxOcrPages, resolvePdfPageIterationLimit } from './ocrLimits'
 
 const moduleRequire = createRequire(path.join(process.cwd(), 'package.json'))
 const pdfJsPackageRoot = path.dirname(moduleRequire.resolve('pdfjs-dist/package.json'))
 const cMapUrl = `${path.join(pdfJsPackageRoot, 'cmaps')}${path.sep}`
 const standardFontDataUrl = `${path.join(pdfJsPackageRoot, 'standard_fonts')}${path.sep}`
+const logger = createLogger('attachments').child({ component: 'pdf-processing' })
 
 const MIN_RENDER_SCALE = 2
 const MAX_RENDER_SCALE = 4
@@ -113,8 +116,17 @@ export async function preparePdfPagesForOcr(filePath: string): Promise<PdfOcrPre
 
   try {
     const pages: PdfPageOcrInput[] = []
+    const maxPages = resolveMaxOcrPages()
+    const pageLimit = resolvePdfPageIterationLimit(pdfDocument.numPages, maxPages)
+    if (pdfDocument.numPages > pageLimit) {
+      logger.warn('PDF page count exceeds OCR cap; truncating', {
+        numPages: pdfDocument.numPages,
+        maxPages: pageLimit,
+        filePath,
+      })
+    }
 
-    for (let pageNumber = 1; pageNumber <= pdfDocument.numPages; pageNumber += 1) {
+    for (let pageNumber = 1; pageNumber <= pageLimit; pageNumber += 1) {
       const page = await pdfDocument.getPage(pageNumber)
 
       try {
@@ -141,7 +153,7 @@ export async function preparePdfPagesForOcr(filePath: string): Promise<PdfOcrPre
     }
 
     return {
-      pageCount: pdfDocument.numPages,
+      pageCount: pages.length,
       pages,
     }
   } finally {

package/src/modules/attachments/lib/scoped-upload-service.ts

@@ -265,9 +265,11 @@ export class ScopedAttachmentUploadService {
     }
 
     if (useLlmOcr) {
-      requestOcrProcessing(em, attachment, driver, storedPath).catch((error) => {
+      try {
+        await requestOcrProcessing(em, attachment, driver, storedPath)
+      } catch (error) {
         logger.error('Scoped attachment OCR scheduling failed', { err: error })
-      })
+      }
     }
     if (this.deps.dataEngine) {
       await emitCrudSideEffects({

package/src/modules/attachments/lib/textExtraction.ts

@@ -1,6 +1,10 @@
 import fs from 'fs/promises'
 import path from 'path'
 import { createRequire } from 'module'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveMaxOcrPages, resolvePdfPageIterationLimit } from './ocrLimits'
+
+const logger = createLogger('attachments').child({ component: 'text-extraction' })
 
 // NOTE: child_process is intentionally NOT imported here.
 // This module MUST NOT shell out to any external binary for content extraction.
@@ -68,8 +72,17 @@ async function extractPdfText(filePath: string): Promise<string | null> {
     })
     const pdfDocument = await loadingTask.promise
     const textParts: string[] = []
+    const maxPages = resolveMaxOcrPages()
+    const pageLimit = resolvePdfPageIterationLimit(pdfDocument.numPages, maxPages)
+    if (pdfDocument.numPages > pageLimit) {
+      logger.warn('PDF page count exceeds text-extraction cap; truncating', {
+        numPages: pdfDocument.numPages,
+        maxPages: pageLimit,
+        filePath,
+      })
+    }
     try {
-      for (let pageNumber = 1; pageNumber <= pdfDocument.numPages; pageNumber += 1) {
+      for (let pageNumber = 1; pageNumber <= pageLimit; pageNumber += 1) {
         const page = await pdfDocument.getPage(pageNumber)
         try {
           const textContent = await page.getTextContent()

package/src/modules/catalog/search.ts

@@ -131,10 +131,14 @@ function buildProductPresenter(
   ) ?? label
   const isActive = record.is_active ?? record.isActive
   const statusText = isActive === false ? translate('catalog.search.status.inactive', 'Inactive') : null
+  const productType = readRecordText(record, 'product_type', 'productType')
+  const productTypeLabel = productType
+    ? translate(`catalog.products.types.${productType}`, productType)
+    : null
   const subtitle = formatSubtitle(
     readRecordText(record, 'subtitle'),
     readRecordText(record, 'sku'),
-    readRecordText(record, 'product_type', 'productType'),
+    productTypeLabel,
     statusText,
   ) ?? snippet(readRecordText(record, 'description'))
   return { title, subtitle, icon: 'package', badge: label }

package/src/modules/communication_channels/api/get/oauth/[provider]/callback/route.ts

@@ -15,6 +15,7 @@ import {
   verifyOAuthState,
   type OAuthStateConsumeStore,
 } from '../../../../../lib/oauth-state'
+import { OAUTH_RESULT_QUERY_PARAM } from '../../../../../lib/oauth-result'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
 const logger = createLogger('communication_channels').child({ component: 'oauth-callback' })
@@ -51,7 +52,7 @@ function redirectWithFlash(
     normalizeOAuthReturnUrl(returnUrl, DEFAULT_OAUTH_RETURN_URL),
     getAppBaseUrl(req),
   )
-  base.searchParams.set('flash', flash.type)
+  base.searchParams.set(OAUTH_RESULT_QUERY_PARAM, flash.type)
   if (flash.code) base.searchParams.set('code', flash.code)
   if (flash.provider) base.searchParams.set('provider', flash.provider)
   if (flash.channelId) base.searchParams.set('channelId', flash.channelId)

package/src/modules/communication_channels/backend/profile/communication-channels/page.tsx

@@ -3,6 +3,7 @@
 import * as React from 'react'
 import { extensionPoints } from '@open-mercato/core/modules/communication_channels/extension-points'
 import { getImportHistoryLimits } from '@open-mercato/core/modules/communication_channels/lib/import-history-limits'
+import { OAUTH_RESULT_QUERY_PARAM, stripOAuthResultParams } from '@open-mercato/core/modules/communication_channels/lib/oauth-result'
 import type { LegacyColumnDef as ColumnDef } from '@tanstack/react-table/legacy'
 import { useRouter, useSearchParams } from 'next/navigation'
 import { Page, PageBody } from '@open-mercato/ui/backend/Page'
@@ -76,7 +77,7 @@ export default function ProfileCommunicationChannelsPage() {
   const t = useT()
   const router = useRouter()
   const searchParams = useSearchParams()
-  const flashType = searchParams?.get('flash')
+  const flashType = searchParams?.get(OAUTH_RESULT_QUERY_PARAM)
   const flashCode = searchParams?.get('code')
   const flashProvider = searchParams?.get('provider')
 
@@ -96,10 +97,13 @@ export default function ProfileCommunicationChannelsPage() {
 
   React.useEffect(() => {
     if (flashType === 'connected') {
+      const providerLabel = flashProvider
+        ? t(`communication_channels.channel.providers.${flashProvider}`, '')
+        : ''
       flash(
-        flashProvider
+        providerLabel
           ? t('communication_channels.profile.flash.connectedWithProvider', 'Channel connected ({provider}).', {
-              provider: flashProvider,
+              provider: providerLabel,
             })
           : t('communication_channels.profile.flash.connected', 'Channel connected.'),
         'success',
@@ -126,15 +130,14 @@ export default function ProfileCommunicationChannelsPage() {
                     'communication_channels.profile.flash.stateStoreUnavailable',
                     'A server error prevented completing the connection. Please try again in a moment.',
                   )
-                : flashCode
-                  ? t('communication_channels.profile.flash.errorWithCode', 'Failed to connect channel — {code}.', {
-                      code: flashCode,
-                    })
-                  : t('communication_channels.profile.flash.error', 'Failed to connect channel.'),
+                : t('communication_channels.profile.flash.error', 'Failed to connect channel.'),
         'error',
       )
+    } else {
+      return
     }
-  }, [flashType, flashCode, flashProvider, t])
+    router.replace(stripOAuthResultParams(window.location.href), { scroll: false })
+  }, [flashType, flashCode, flashProvider, router, t])
 
   React.useEffect(() => {
     let cancelled = false

package/src/modules/communication_channels/i18n/de.json

@@ -107,7 +107,6 @@
   "communication_channels.profile.flash.connected": "Kanal verbunden.",
   "communication_channels.profile.flash.connectedWithProvider": "Kanal verbunden ({provider}).",
   "communication_channels.profile.flash.error": "Kanal konnte nicht verbunden werden.",
-  "communication_channels.profile.flash.errorWithCode": "Kanal konnte nicht verbunden werden — {code}.",
   "communication_channels.profile.flash.replayedState": "Dieser Verbindungslink wurde bereits verwendet — bitte starten Sie die Verbindung erneut.",
   "communication_channels.profile.flash.stateStoreUnavailable": "Ein Serverfehler hat die Verbindung verhindert. Bitte versuchen Sie es in Kürze erneut.",
   "communication_channels.profile.group": "Profil",

package/src/modules/communication_channels/i18n/en.json

@@ -107,7 +107,6 @@
   "communication_channels.profile.flash.connected": "Channel connected.",
   "communication_channels.profile.flash.connectedWithProvider": "Channel connected ({provider}).",
   "communication_channels.profile.flash.error": "Failed to connect channel.",
-  "communication_channels.profile.flash.errorWithCode": "Failed to connect channel — {code}.",
   "communication_channels.profile.flash.replayedState": "This connection link was already used — please start the connection again.",
   "communication_channels.profile.flash.stateStoreUnavailable": "A server error prevented completing the connection. Please try again in a moment.",
   "communication_channels.profile.group": "Profile",

package/src/modules/communication_channels/i18n/es.json

@@ -107,7 +107,6 @@
   "communication_channels.profile.flash.connected": "Canal conectado.",
   "communication_channels.profile.flash.connectedWithProvider": "Canal conectado ({provider}).",
   "communication_channels.profile.flash.error": "No se pudo conectar el canal.",
-  "communication_channels.profile.flash.errorWithCode": "No se pudo conectar el canal — {code}.",
   "communication_channels.profile.flash.replayedState": "Este enlace de conexión ya se utilizó — inicia la conexión de nuevo.",
   "communication_channels.profile.flash.stateStoreUnavailable": "Un error del servidor impidió completar la conexión. Inténtalo de nuevo en un momento.",
   "communication_channels.profile.group": "Perfil",

package/src/modules/communication_channels/i18n/ko.json

@@ -107,7 +107,6 @@
   "communication_channels.profile.flash.connected": "채널이 연결되었습니다.",
   "communication_channels.profile.flash.connectedWithProvider": "채널이 연결되었습니다 ({provider}).",
   "communication_channels.profile.flash.error": "채널 연결에 실패했습니다.",
-  "communication_channels.profile.flash.errorWithCode": "채널 연결에 실패했습니다 — {code}.",
   "communication_channels.profile.flash.replayedState": "이 연결 링크는 이미 사용되었습니다. 연결을 다시 시작해 주세요.",
   "communication_channels.profile.flash.stateStoreUnavailable": "서버 오류로 연결을 완료할 수 없습니다. 잠시 후 다시 시도해 주세요.",
   "communication_channels.profile.group": "프로필",

package/src/modules/communication_channels/i18n/pl.json

@@ -107,7 +107,6 @@
   "communication_channels.profile.flash.connected": "Kanał połączony.",
   "communication_channels.profile.flash.connectedWithProvider": "Kanał połączony ({provider}).",
   "communication_channels.profile.flash.error": "Nie udało się połączyć kanału.",
-  "communication_channels.profile.flash.errorWithCode": "Nie udało się połączyć kanału — {code}.",
   "communication_channels.profile.flash.replayedState": "Ten link połączenia został już użyty — rozpocznij połączenie ponownie.",
   "communication_channels.profile.flash.stateStoreUnavailable": "Błąd serwera uniemożliwił dokończenie połączenia. Spróbuj ponownie za chwilę.",
   "communication_channels.profile.group": "Profil",

package/src/modules/communication_channels/lib/oauth-result.ts

@@ -0,0 +1,13 @@
+// The OAuth callback reports its outcome to the return page through this query
+// param. It must not be `flash`: the global <FlashMessages> host reads `?flash=`
+// as the message text and would render the raw outcome ("error"/"connected") as
+// a success toast (#6402).
+export const OAUTH_RESULT_QUERY_PARAM = 'oauth'
+
+const OAUTH_RESULT_QUERY_PARAMS = [OAUTH_RESULT_QUERY_PARAM, 'code', 'provider', 'channelId']
+
+export function stripOAuthResultParams(href: string): string {
+  const url = new URL(href)
+  for (const param of OAUTH_RESULT_QUERY_PARAMS) url.searchParams.delete(param)
+  return `${url.pathname}${url.search}${url.hash}`
+}