@open-mercato/core 0.6.6-develop.5834.1.dc9eb0615e → 0.6.6-develop.5900.1.b3fb52925e

package/src/modules/query_index/lib/engine.ts

@@ -1,4 +1,4 @@
-import type { QueryEngine, QueryOptions, QueryResult, FilterOp, Filter, QueryCustomFieldSource, PartialIndexWarning, QueryExtensionsConfig, Sort } from '@open-mercato/shared/lib/query/types'
+import type { QueryEngine, QueryOptions, QueryResult, QueryResultMeta, EncryptedSortRowCapWarning, FilterOp, Filter, QueryCustomFieldSource, PartialIndexWarning, QueryExtensionsConfig, Sort } from '@open-mercato/shared/lib/query/types'
 import { SortDir } from '@open-mercato/shared/lib/query/types'
 import type { EntityId } from '@open-mercato/shared/modules/entities'
 import type { EntityManager } from '@mikro-orm/postgresql'
@@ -21,7 +21,10 @@ import {
 import { resolveSearchConfig, type SearchConfig } from '@open-mercato/shared/lib/search/config'
 import { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from '@open-mercato/shared/lib/query/query-extension-runner'
-import { resolveEncryptedSortFields, sortRowsInMemory } from '@open-mercato/shared/lib/query/encrypted-sort'
+import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from '@open-mercato/shared/lib/query/encrypted-sort'
+import { mapWithConcurrency } from '@open-mercato/shared/lib/query/bounded-decrypt'
+
+const DECRYPT_CONCURRENCY = 8
 
 function buildFilterableCustomFieldJoins(
   sources: QueryCustomFieldSource[] | undefined,
@@ -795,9 +798,10 @@ export class HybridQueryEngine implements QueryEngine {
       }
       const selectFields = Array.from(selectFieldSet)
 
-      const applySelection = (q: AnyBuilder): AnyBuilder => {
+      const applySelection = (q: AnyBuilder, fieldsOverride?: string[]): AnyBuilder => {
         let next = q
-        for (const field of selectFields) {
+        const fields = fieldsOverride ?? selectFields
+        for (const field of fields) {
           const fieldName = String(field)
           if (fieldName.startsWith('cf:')) {
             const alias = this.sanitize(fieldName)
@@ -888,71 +892,131 @@ export class HybridQueryEngine implements QueryEngine {
         total = this.parseCount(countRow)
       }
 
-      const dataRoot = db.selectFrom(`${baseTable} as b` as any)
-      let dataBuilder = await applyQueryShape(dataRoot)
-      dataBuilder = applySelection(dataBuilder)
-      dataBuilder = applySort(dataBuilder)
-      if (!requiresPlaintextSort) {
-        dataBuilder = dataBuilder.limit(pageSize).offset((page - 1) * pageSize)
-      }
+      const dekKeyCache = new Map<string | null, string | null>()
 
-      if (debugEnabled && sqlDebugEnabled) {
-        const compiled = dataBuilder.compile()
-        this.debug('query:sql:data', { entity, sql: compiled.sql, bindings: compiled.parameters, page, pageSize })
+      const decryptRow = async (item: Record<string, unknown>): Promise<Record<string, unknown>> => {
+        let next = item
+        if (encSvc?.decryptEntityPayload) {
+          const decrypt = encSvc.decryptEntityPayload.bind(encSvc) as (
+            entityId: EntityId, payload: Record<string, unknown>, tenantId: string | null, organizationId: string | null,
+          ) => Promise<Record<string, unknown>>
+          try {
+            const decrypted = await decrypt(
+              entity, next,
+              (next?.tenant_id ?? next?.tenantId ?? opts.tenantId ?? null) as string | null,
+              (next?.organization_id ?? next?.organizationId ?? fallbackOrgId ?? null) as string | null,
+            )
+            next = { ...next, ...decrypted }
+          } catch (err) {
+            console.error('Error decrypting entity payload', err)
+          }
+        }
+        if (encSvc) {
+          try {
+            next = await decryptIndexDocCustomFields(
+              next,
+              {
+                tenantId: (next?.tenant_id ?? next?.tenantId ?? opts.tenantId ?? null) as string | null,
+                organizationId: (next?.organization_id ?? next?.organizationId ?? null) as string | null,
+              },
+              encSvc as any, dekKeyCache,
+            )
+          } catch { /* keep next as-is */ }
+        }
+        return next
       }
-      const itemsRaw = await this.captureSqlTiming(
-        'query:sql:data', entity,
-        () => dataBuilder.execute(),
-        { page, pageSize }, profiler,
-      )
-      if (debugEnabled) this.debug('query:complete', { entity, total, items: Array.isArray(itemsRaw) ? itemsRaw.length : 0 })
 
-      let items = itemsRaw as any[]
-      const dekKeyCache = new Map<string | null, string | null>()
-      if (encSvc?.decryptEntityPayload) {
-        const decrypt = encSvc.decryptEntityPayload.bind(encSvc) as (
-          entityId: EntityId, payload: Record<string, unknown>, tenantId: string | null, organizationId: string | null,
-        ) => Promise<Record<string, unknown>>
-        items = await Promise.all(
-          items.map(async (item) => {
-            try {
-              const decrypted = await decrypt(
-                entity, item,
-                item?.tenant_id ?? item?.tenantId ?? opts.tenantId ?? null,
-                item?.organization_id ?? item?.organizationId ?? fallbackOrgId ?? null,
-              )
-              return { ...item, ...decrypted }
-            } catch (err) {
-              console.error('Error decrypting entity payload', err)
-              return item
-            }
-          })
-        )
-      }
-      if (encSvc) {
-        items = await Promise.all(
-          items.map(async (item) => {
-            try {
-              return await decryptIndexDocCustomFields(
-                item,
-                {
-                  tenantId: item?.tenant_id ?? item?.tenantId ?? opts.tenantId ?? null,
-                  organizationId: item?.organization_id ?? item?.organizationId ?? null,
-                },
-                encSvc as any, dekKeyCache,
-              )
-            } catch { return item }
-          }),
-        )
-      }
+      let items: Record<string, unknown>[]
+      let encryptedSortRowCapWarning: EncryptedSortRowCapWarning | undefined
+
       if (requiresPlaintextSort) {
-        items = sortRowsInMemory(items as Record<string, unknown>[], resolvedSorts)
+        // Phase 1: slim id+sort-columns candidate scan, bounded-concurrency decrypt,
+        // in-memory sort over the full candidate set, then slice to the page's ids.
+        const cap = resolveEncryptedSortMaxRows()
+        const phase1Root = db.selectFrom(`${baseTable} as b` as any)
+        let phase1 = await applyQueryShape(phase1Root)
+        const sortFieldNames = Array.from(new Set(['id', ...resolvedSorts.map((s) => String(s.field))]))
+        phase1 = applySelection(phase1, sortFieldNames)
+        if (cap !== null) {
+          phase1 = phase1.limit(cap).orderBy(qualify('id'), 'asc' as any)
+        }
+        if (debugEnabled && sqlDebugEnabled) {
+          const compiled = phase1.compile()
+          this.debug('query:sql:data:phase1', { entity, sql: compiled.sql, bindings: compiled.parameters })
+        }
+        const candidateRows = await this.captureSqlTiming(
+          'query:sql:data:phase1', entity,
+          () => phase1.execute(),
+          { phase: 1 }, profiler,
+        ) as Record<string, unknown>[]
+        const decryptedCandidates = await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow)
+        const orderedCandidates = sortRowsInMemory(decryptedCandidates, resolvedSorts)
+        const pageIds = orderedCandidates
           .slice((page - 1) * pageSize, page * pageSize)
+          .map((row) => row.id)
+
+        if (cap !== null && total > cap) {
+          encryptedSortRowCapWarning = {
+            entity,
+            sortFields: resolvedSorts.map((s) => String(s.field)),
+            maxRows: cap,
+            totalMatched: total,
+          }
+        }
+
+        // Phase 2: fetch + decrypt full rows for just the page's ids, then reassemble
+        // in phase-1's order (SQL `WHERE id IN (...)` order is unspecified — never
+        // trust it for ordering).
+        if (pageIds.length === 0) {
+          items = []
+        } else {
+          const phase2Root = db.selectFrom(`${baseTable} as b` as any)
+          let phase2 = await applyQueryShape(phase2Root)
+          phase2 = applySelection(phase2)
+          phase2 = phase2.where(qualify('id'), 'in', pageIds)
+          if (debugEnabled && sqlDebugEnabled) {
+            const compiled = phase2.compile()
+            this.debug('query:sql:data:phase2', { entity, sql: compiled.sql, bindings: compiled.parameters })
+          }
+          const pageRowsRaw = await this.captureSqlTiming(
+            'query:sql:data:phase2', entity,
+            () => phase2.execute(),
+            { phase: 2 }, profiler,
+          ) as Record<string, unknown>[]
+          const decryptedPageRows = await mapWithConcurrency(pageRowsRaw, DECRYPT_CONCURRENCY, decryptRow)
+          const byId = new Map(decryptedPageRows.map((row) => [String(row.id), row]))
+          items = pageIds
+            .map((id) => byId.get(String(id)))
+            .filter((row): row is Record<string, unknown> => row != null)
+        }
+      } else {
+        const dataRoot = db.selectFrom(`${baseTable} as b` as any)
+        let dataBuilder = await applyQueryShape(dataRoot)
+        dataBuilder = applySelection(dataBuilder)
+        dataBuilder = applySort(dataBuilder)
+        dataBuilder = dataBuilder.limit(pageSize).offset((page - 1) * pageSize)
+
+        if (debugEnabled && sqlDebugEnabled) {
+          const compiled = dataBuilder.compile()
+          this.debug('query:sql:data', { entity, sql: compiled.sql, bindings: compiled.parameters, page, pageSize })
+        }
+        const itemsRaw = await this.captureSqlTiming(
+          'query:sql:data', entity,
+          () => dataBuilder.execute(),
+          { page, pageSize }, profiler,
+        ) as Record<string, unknown>[]
+        items = await mapWithConcurrency(itemsRaw, DECRYPT_CONCURRENCY, decryptRow)
       }
+      if (debugEnabled) this.debug('query:complete', { entity, total, items: items.length })
 
       const typedItems = items as unknown as T[]
       let result: QueryResult<T> = { items: typedItems, page, pageSize, total }
-      if (partialIndexWarning) result.meta = { partialIndexWarning }
+      if (partialIndexWarning || encryptedSortRowCapWarning) {
+        const meta: QueryResultMeta = {}
+        if (partialIndexWarning) meta.partialIndexWarning = partialIndexWarning
+        if (encryptedSortRowCapWarning) meta.encryptedSortRowCapWarning = encryptedSortRowCapWarning
+        result.meta = meta
+      }
 
       result = await applyAfterExtensions(result)
       finishProfile({

package/src/modules/sync_excel/widgets/injection/upload-config/widget.client.tsx

@@ -12,6 +12,7 @@ import { apiCall } from '@open-mercato/ui/backend/utils/apiCall'
 import { useCustomFieldDefs } from '@open-mercato/ui/backend/utils/customFieldDefs'
 import { Alert, AlertDescription, AlertTitle } from '@open-mercato/ui/primitives/alert'
 import { Badge } from '@open-mercato/ui/primitives/badge'
+import { StatusBadge, type StatusMap } from '@open-mercato/ui/primitives/status-badge'
 import { Button } from '@open-mercato/ui/primitives/button'
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@open-mercato/ui/primitives/card'
 import { Input } from '@open-mercato/ui/primitives/input'
@@ -1171,54 +1172,67 @@ function MetricCard({ label, value }: { label: string; value: string }) {
   )
 }
 
-function RunStatusBadge({
+type RunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled' | 'idle'
+
+export const RUN_STATUS_BADGE_VARIANTS: StatusMap<RunStatus> = {
+  completed: 'success',
+  failed: 'error',
+  cancelled: 'neutral',
+  pending: 'info',
+  running: 'info',
+  idle: 'neutral',
+}
+
+export function RunStatusBadge({
   status,
   t,
 }: {
-  status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled' | 'idle'
+  status: RunStatus
   t: ReturnType<typeof useT>
 }) {
+  const variant = RUN_STATUS_BADGE_VARIANTS[status] ?? 'neutral'
+
   if (status === 'completed') {
     return (
-      <Badge variant="outline" className="gap-1.5 border-emerald-500/30 bg-emerald-500/10 text-emerald-300">
+      <StatusBadge variant={variant} className="gap-1.5">
         <CheckCircle2 className="size-3.5" />
         {t('sync_excel.widget.run.status.completed', 'Completed')}
-      </Badge>
+      </StatusBadge>
     )
   }
 
   if (status === 'failed') {
     return (
-      <Badge variant="outline" className="gap-1.5 border-red-500/30 bg-red-500/10 text-red-300">
+      <StatusBadge variant={variant} className="gap-1.5">
         <AlertTriangle className="size-3.5" />
         {t('sync_excel.widget.run.status.failed', 'Failed')}
-      </Badge>
+      </StatusBadge>
     )
   }
 
   if (status === 'cancelled') {
     return (
-      <Badge variant="outline" className="gap-1.5 border-zinc-500/30 bg-zinc-500/10 text-zinc-300">
+      <StatusBadge variant={variant} className="gap-1.5">
         <XCircle className="size-3.5" />
         {t('sync_excel.widget.run.status.cancelled', 'Cancelled')}
-      </Badge>
+      </StatusBadge>
     )
   }
 
   if (status === 'pending' || status === 'running') {
     return (
-      <Badge variant="outline" className="gap-1.5 border-blue-500/30 bg-blue-500/10 text-blue-300">
+      <StatusBadge variant={variant} className="gap-1.5">
         <RefreshCw className={cn('size-3.5', status === 'running' ? 'animate-spin' : '')} />
         {status === 'pending'
           ? t('sync_excel.widget.run.status.pending', 'Pending')
           : t('sync_excel.widget.run.status.running', 'Running')}
-      </Badge>
+      </StatusBadge>
     )
   }
 
   return (
-    <Badge variant="outline" className="gap-1.5 border-zinc-500/30 bg-zinc-500/10 text-zinc-300">
+    <StatusBadge variant={variant}>
       {t('sync_excel.widget.run.status.idle', 'Idle')}
-    </Badge>
+    </StatusBadge>
   )
 }