@open-mercato/core 0.9.0 → 0.9.1-develop.7462.1.94c09f158f

package/src/modules/attachments/lib/ocrQueue.ts

@@ -3,6 +3,8 @@ import { Attachment, AttachmentPartition } from '../data/entities'
 import { OcrService } from './ocrService'
 import type { StorageDriver } from './drivers/types'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveOcrMaxConcurrency, resolveOcrMaxWaitQueue } from './ocrLimits'
+import { extractAttachmentContent } from './textExtraction'
 
 const logger = createLogger('attachments').child({ component: 'ocr' })
 
@@ -15,6 +17,45 @@ export type OcrRequestedEvent = {
   tenantId: string | null
 }
 
+/** Outcome of `requestOcrProcessing` — background LLM OCR vs inline text fallback. */
+export type OcrProcessingDispatch = 'queued' | 'inline_fallback'
+
+let activeOcrJobs = 0
+const ocrWaitQueue: Array<() => void> = []
+// Jobs scheduled with setImmediate that have not yet entered a slot or the wait queue.
+// Counted toward the wait-queue cap so a burst in one event-loop turn cannot pass the check together.
+let pendingOcrJobs = 0
+
+/** Test-only: reset in-process OCR concurrency bookkeeping. */
+export function resetOcrConcurrencyStateForTests(): void {
+  activeOcrJobs = 0
+  ocrWaitQueue.length = 0
+  pendingOcrJobs = 0
+}
+
+/** Test-only: inspect in-process OCR concurrency counters. */
+export function getOcrConcurrencyStateForTests(): { active: number; waiting: number } {
+  return { active: activeOcrJobs, waiting: ocrWaitQueue.length }
+}
+
+export async function withOcrConcurrencySlot<T>(run: () => Promise<T>): Promise<T> {
+  const maxConcurrency = resolveOcrMaxConcurrency()
+  // Re-check after wake: multiple waiters can resume when one slot frees (barging).
+  while (activeOcrJobs >= maxConcurrency) {
+    await new Promise<void>((resolve) => {
+      ocrWaitQueue.push(resolve)
+    })
+  }
+  activeOcrJobs += 1
+  try {
+    return await run()
+  } finally {
+    activeOcrJobs -= 1
+    const next = ocrWaitQueue.shift()
+    if (next) next()
+  }
+}
+
 export async function processAttachmentOcr(
   em: EntityManager,
   payload: OcrRequestedEvent,
@@ -73,12 +114,49 @@ export async function processAttachmentOcr(
   }
 }
 
+/**
+ * When the in-process OCR wait queue is full, extract plain text/PDF/DOCX content
+ * inline (no LLM) so `attachment.content` is not left NULL forever.
+ */
+async function persistInlineTextExtractionFallback(
+  em: EntityManager,
+  payload: OcrRequestedEvent,
+  driver: StorageDriver,
+): Promise<void> {
+  const { attachmentId, storagePath, mimeType, partitionCode } = payload
+  const { filePath, cleanup } = await driver.toLocalPath(partitionCode, storagePath)
+  try {
+    const content = await extractAttachmentContent({ filePath, mimeType })
+    if (!content) {
+      logger.info('OCR wait queue full; inline extraction produced no content', { attachmentId })
+      return
+    }
+    const row = await em.findOne(Attachment, { id: attachmentId })
+    if (!row) {
+      logger.error('Attachment not found during OCR overflow fallback', { attachmentId })
+      return
+    }
+    row.content = content
+    await em.persist(row).flush()
+    logger.info('OCR wait queue full; stored inline text extraction', {
+      attachmentId,
+      contentLength: content.length,
+    })
+  } catch (error) {
+    logger.error('OCR wait queue overflow inline extraction failed', { attachmentId, err: error })
+  } finally {
+    await cleanup().catch((cleanupError) => {
+      logger.warn('Temp file cleanup failed after OCR overflow fallback', { err: cleanupError })
+    })
+  }
+}
+
 export async function requestOcrProcessing(
   em: EntityManager,
   attachment: Attachment,
   driver: StorageDriver,
   storagePath: string,
-): Promise<void> {
+): Promise<OcrProcessingDispatch> {
   const payload: OcrRequestedEvent = {
     attachmentId: attachment.id,
     storagePath,
@@ -97,9 +175,24 @@ export async function requestOcrProcessing(
 
   const workerEm = em.fork()
 
+  const maxWaitQueue = resolveOcrMaxWaitQueue()
+  if (ocrWaitQueue.length + pendingOcrJobs >= maxWaitQueue) {
+    logger.warn('OCR wait queue full; falling back to inline text extraction', {
+      attachmentId: attachment.id,
+      waiting: ocrWaitQueue.length,
+      pending: pendingOcrJobs,
+      maxWaitQueue,
+    })
+    await persistInlineTextExtractionFallback(workerEm, payload, driver)
+    return 'inline_fallback'
+  }
+
+  pendingOcrJobs += 1
   setImmediate(() => {
-    processAttachmentOcr(workerEm, payload, driver).catch((error) => {
+    pendingOcrJobs -= 1
+    withOcrConcurrencySlot(() => processAttachmentOcr(workerEm, payload, driver)).catch((error) => {
       logger.error('Background processing error', { err: error })
     })
   })
+  return 'queued'
 }

package/src/modules/attachments/lib/ocrService.ts

@@ -3,6 +3,7 @@ import { createOpenAI } from '@ai-sdk/openai'
 import fs from 'fs/promises'
 import path from 'path'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveOcrMaxOutputTokens, resolveOcrPageTimeoutMs } from './ocrLimits'
 
 const logger = createLogger('attachments').child({ component: 'ocr-service' })
 
@@ -102,6 +103,9 @@ export class OcrService {
     try {
       const result = await generateText({
         model: client(model),
+        abortSignal: AbortSignal.timeout(resolveOcrPageTimeoutMs()),
+        maxRetries: 0,
+        maxOutputTokens: resolveOcrMaxOutputTokens(),
         messages: [
           {
             role: 'user',

package/src/modules/attachments/lib/pdfProcessing.ts

@@ -1,11 +1,14 @@
 import fs from 'fs/promises'
 import path from 'path'
 import { createRequire } from 'module'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveMaxOcrPages, resolvePdfPageIterationLimit } from './ocrLimits'
 
 const moduleRequire = createRequire(path.join(process.cwd(), 'package.json'))
 const pdfJsPackageRoot = path.dirname(moduleRequire.resolve('pdfjs-dist/package.json'))
 const cMapUrl = `${path.join(pdfJsPackageRoot, 'cmaps')}${path.sep}`
 const standardFontDataUrl = `${path.join(pdfJsPackageRoot, 'standard_fonts')}${path.sep}`
+const logger = createLogger('attachments').child({ component: 'pdf-processing' })
 
 const MIN_RENDER_SCALE = 2
 const MAX_RENDER_SCALE = 4
@@ -113,8 +116,17 @@ export async function preparePdfPagesForOcr(filePath: string): Promise<PdfOcrPre
 
   try {
     const pages: PdfPageOcrInput[] = []
+    const maxPages = resolveMaxOcrPages()
+    const pageLimit = resolvePdfPageIterationLimit(pdfDocument.numPages, maxPages)
+    if (pdfDocument.numPages > pageLimit) {
+      logger.warn('PDF page count exceeds OCR cap; truncating', {
+        numPages: pdfDocument.numPages,
+        maxPages: pageLimit,
+        filePath,
+      })
+    }
 
-    for (let pageNumber = 1; pageNumber <= pdfDocument.numPages; pageNumber += 1) {
+    for (let pageNumber = 1; pageNumber <= pageLimit; pageNumber += 1) {
       const page = await pdfDocument.getPage(pageNumber)
 
       try {
@@ -141,7 +153,7 @@ export async function preparePdfPagesForOcr(filePath: string): Promise<PdfOcrPre
     }
 
     return {
-      pageCount: pdfDocument.numPages,
+      pageCount: pages.length,
       pages,
     }
   } finally {

package/src/modules/attachments/lib/scoped-upload-service.ts

@@ -265,9 +265,11 @@ export class ScopedAttachmentUploadService {
     }
 
     if (useLlmOcr) {
-      requestOcrProcessing(em, attachment, driver, storedPath).catch((error) => {
+      try {
+        await requestOcrProcessing(em, attachment, driver, storedPath)
+      } catch (error) {
         logger.error('Scoped attachment OCR scheduling failed', { err: error })
-      })
+      }
     }
     if (this.deps.dataEngine) {
       await emitCrudSideEffects({

package/src/modules/attachments/lib/textExtraction.ts

@@ -1,6 +1,10 @@
 import fs from 'fs/promises'
 import path from 'path'
 import { createRequire } from 'module'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import { resolveMaxOcrPages, resolvePdfPageIterationLimit } from './ocrLimits'
+
+const logger = createLogger('attachments').child({ component: 'text-extraction' })
 
 // NOTE: child_process is intentionally NOT imported here.
 // This module MUST NOT shell out to any external binary for content extraction.
@@ -68,8 +72,17 @@ async function extractPdfText(filePath: string): Promise<string | null> {
     })
     const pdfDocument = await loadingTask.promise
     const textParts: string[] = []
+    const maxPages = resolveMaxOcrPages()
+    const pageLimit = resolvePdfPageIterationLimit(pdfDocument.numPages, maxPages)
+    if (pdfDocument.numPages > pageLimit) {
+      logger.warn('PDF page count exceeds text-extraction cap; truncating', {
+        numPages: pdfDocument.numPages,
+        maxPages: pageLimit,
+        filePath,
+      })
+    }
     try {
-      for (let pageNumber = 1; pageNumber <= pdfDocument.numPages; pageNumber += 1) {
+      for (let pageNumber = 1; pageNumber <= pageLimit; pageNumber += 1) {
         const page = await pdfDocument.getPage(pageNumber)
         try {
           const textContent = await page.getTextContent()

package/src/modules/auth/i18n/de.json

@@ -212,6 +212,8 @@
   "auth.acl.features.forms.view": "Formulare und Einreichungen ansehen",
   "auth.acl.features.gateway_stripe.configure": "Configure Stripe gateway settings",
   "auth.acl.features.gateway_stripe.view": "View Stripe gateway configuration",
+  "auth.acl.features.gateway_tpay.configure": "Tpay-Gateway-Einstellungen konfigurieren",
+  "auth.acl.features.gateway_tpay.view": "Tpay-Gateway-Konfiguration anzeigen",
   "auth.acl.features.inbox_ops.log.view": "View processing log",
   "auth.acl.features.inbox_ops.proposals.manage": "Manage proposals",
   "auth.acl.features.inbox_ops.proposals.view": "View proposals",
@@ -428,6 +430,7 @@
   "auth.acl.modules.example_customers_sync": "Example Customers Sync",
   "auth.acl.modules.feature_toggles": "Feature Toggles",
   "auth.acl.modules.gateway_stripe": "Stripe Payment Gateway",
+  "auth.acl.modules.gateway_tpay": "Tpay-Zahlungsgateway",
   "auth.acl.modules.inbox_ops": "InboxOps — Email-to-ERP Agent",
   "auth.acl.modules.integrations": "Integrations",
   "auth.acl.modules.messages": "Messages",

package/src/modules/auth/i18n/en.json

@@ -212,6 +212,8 @@
   "auth.acl.features.forms.view": "View forms and submissions",
   "auth.acl.features.gateway_stripe.configure": "Configure Stripe gateway settings",
   "auth.acl.features.gateway_stripe.view": "View Stripe gateway configuration",
+  "auth.acl.features.gateway_tpay.configure": "Configure Tpay gateway settings",
+  "auth.acl.features.gateway_tpay.view": "View Tpay gateway configuration",
   "auth.acl.features.inbox_ops.log.view": "View processing log",
   "auth.acl.features.inbox_ops.proposals.manage": "Manage proposals",
   "auth.acl.features.inbox_ops.proposals.view": "View proposals",
@@ -428,6 +430,7 @@
   "auth.acl.modules.example_customers_sync": "Example Customers Sync",
   "auth.acl.modules.feature_toggles": "Feature Toggles",
   "auth.acl.modules.gateway_stripe": "Stripe Payment Gateway",
+  "auth.acl.modules.gateway_tpay": "Tpay Payment Gateway",
   "auth.acl.modules.inbox_ops": "InboxOps — Email-to-ERP Agent",
   "auth.acl.modules.integrations": "Integrations",
   "auth.acl.modules.messages": "Messages",

package/src/modules/auth/i18n/es.json

@@ -212,6 +212,8 @@
   "auth.acl.features.forms.view": "Ver formularios y envíos",
   "auth.acl.features.gateway_stripe.configure": "Configure Stripe gateway settings",
   "auth.acl.features.gateway_stripe.view": "View Stripe gateway configuration",
+  "auth.acl.features.gateway_tpay.configure": "Configurar los ajustes de la pasarela Tpay",
+  "auth.acl.features.gateway_tpay.view": "Ver la configuración de la pasarela Tpay",
   "auth.acl.features.inbox_ops.log.view": "View processing log",
   "auth.acl.features.inbox_ops.proposals.manage": "Manage proposals",
   "auth.acl.features.inbox_ops.proposals.view": "View proposals",
@@ -428,6 +430,7 @@
   "auth.acl.modules.example_customers_sync": "Example Customers Sync",
   "auth.acl.modules.feature_toggles": "Feature Toggles",
   "auth.acl.modules.gateway_stripe": "Stripe Payment Gateway",
+  "auth.acl.modules.gateway_tpay": "Pasarela de pago Tpay",
   "auth.acl.modules.inbox_ops": "InboxOps — Email-to-ERP Agent",
   "auth.acl.modules.integrations": "Integrations",
   "auth.acl.modules.messages": "Messages",

package/src/modules/auth/i18n/ko.json

@@ -212,6 +212,8 @@
   "auth.acl.features.forms.view": "View forms and submissions",
   "auth.acl.features.gateway_stripe.configure": "Configure Stripe gateway settings",
   "auth.acl.features.gateway_stripe.view": "View Stripe gateway configuration",
+  "auth.acl.features.gateway_tpay.configure": "Tpay 게이트웨이 설정 구성",
+  "auth.acl.features.gateway_tpay.view": "Tpay 게이트웨이 구성 보기",
   "auth.acl.features.inbox_ops.log.view": "View processing log",
   "auth.acl.features.inbox_ops.proposals.manage": "Manage proposals",
   "auth.acl.features.inbox_ops.proposals.view": "View proposals",
@@ -428,6 +430,7 @@
   "auth.acl.modules.example_customers_sync": "Example Customers Sync",
   "auth.acl.modules.feature_toggles": "Feature Toggles",
   "auth.acl.modules.gateway_stripe": "Stripe Payment Gateway",
+  "auth.acl.modules.gateway_tpay": "Tpay 결제 게이트웨이",
   "auth.acl.modules.inbox_ops": "InboxOps — Email-to-ERP Agent",
   "auth.acl.modules.integrations": "Integrations",
   "auth.acl.modules.messages": "Messages",

package/src/modules/auth/i18n/pl.json

@@ -212,6 +212,8 @@
   "auth.acl.features.forms.view": "Przeglądanie formularzy i zgłoszeń",
   "auth.acl.features.gateway_stripe.configure": "Konfiguruj ustawienia bramki Stripe",
   "auth.acl.features.gateway_stripe.view": "Wyświetl konfigurację bramki Stripe",
+  "auth.acl.features.gateway_tpay.configure": "Konfiguruj ustawienia bramki Tpay",
+  "auth.acl.features.gateway_tpay.view": "Wyświetl konfigurację bramki Tpay",
   "auth.acl.features.inbox_ops.log.view": "Wyświetl dziennik przetwarzania",
   "auth.acl.features.inbox_ops.proposals.manage": "Zarządzaj propozycjami",
   "auth.acl.features.inbox_ops.proposals.view": "Wyświetl propozycje",
@@ -428,6 +430,7 @@
   "auth.acl.modules.example_customers_sync": "Przykładowa synchronizacja klientów",
   "auth.acl.modules.feature_toggles": "Przełączniki funkcji",
   "auth.acl.modules.gateway_stripe": "Bramka płatnicza Stripe",
+  "auth.acl.modules.gateway_tpay": "Bramka płatnicza Tpay",
   "auth.acl.modules.inbox_ops": "InboxOps — agent e-mail do ERP",
   "auth.acl.modules.integrations": "Integracje",
   "auth.acl.modules.messages": "Wiadomości",

package/src/modules/catalog/commands/categories.ts

@@ -455,7 +455,7 @@ const updateCategoryCommand: CommandHandler<CategoryUpdateInput, { categoryId: s
       ],
       { transaction: true }
     )
-    const resetValues = buildCustomFieldResetMap(payload?.after?.custom ?? undefined, before.custom ?? undefined)
+    const resetValues = buildCustomFieldResetMap(before.custom ?? undefined, payload?.after?.custom ?? undefined)
     if (Object.keys(resetValues).length) {
       await setCustomFieldsIfAny({
         dataEngine: ctx.container.resolve('dataEngine'),