@open-mercato/core 0.6.7-develop.6696.1.4236709f7e → 0.6.7-develop.6725.1.07a748cfc2

package/src/modules/customers/lib/dealStatus.ts

@@ -0,0 +1,48 @@
+// Single source of truth for deal open/closed (terminal) semantics.
+//
+// `customer_deals.status` is a lenient text column and the supported writers do not agree on
+// one spelling: the closure hooks and the kanban board persist `win` / `loose`, the AI tool
+// `customers.update_deal_stage` persists `won` / `lost`, and the seeded vocabulary also carries
+// `closed`. The terminal set below therefore covers every spelling a status writer persists, so
+// status-based readers can share one classification. `closureOutcome` is a separate signal and
+// callers that combine both columns must handle it explicitly.
+//
+// A status not known to code counts as OPEN. Deal stages are configurable per tenant, so the
+// lenient column can hold values this module has never seen — treating them as open keeps them
+// visible in active-deal counts instead of silently reclassifying them as closed.
+
+export const DEAL_STATUS_WIN = 'win' as const
+export const DEAL_STATUS_LOSE = 'loose' as const
+export const DEAL_STATUS_CLOSED = 'closed' as const
+
+// `won` / `lost` are the spellings the AI stage tool writes; `win` / `loose` are what the UI
+// closure flows persist. Both are accepted so neither writer produces an unreadable status.
+export const WON_DEAL_STATUS_LIST: readonly string[] = [DEAL_STATUS_WIN, 'won']
+
+export const LOST_DEAL_STATUS_LIST: readonly string[] = [DEAL_STATUS_LOSE, 'lost']
+
+export const CLOSED_DEAL_STATUS_LIST: readonly string[] = [
+  ...WON_DEAL_STATUS_LIST,
+  ...LOST_DEAL_STATUS_LIST,
+  DEAL_STATUS_CLOSED,
+]
+
+const WON_DEAL_STATUSES = new Set<string>(WON_DEAL_STATUS_LIST)
+const LOST_DEAL_STATUSES = new Set<string>(LOST_DEAL_STATUS_LIST)
+const CLOSED_DEAL_STATUSES = new Set<string>(CLOSED_DEAL_STATUS_LIST)
+
+export function isWonDealStatus(value: string | null | undefined): boolean {
+  return value != null && WON_DEAL_STATUSES.has(value)
+}
+
+export function isLostDealStatus(value: string | null | undefined): boolean {
+  return value != null && LOST_DEAL_STATUSES.has(value)
+}
+
+export function isClosedDealStatus(value: string | null | undefined): boolean {
+  return value != null && CLOSED_DEAL_STATUSES.has(value)
+}
+
+export function isOpenDealStatus(value: string | null | undefined): boolean {
+  return !isClosedDealStatus(value)
+}

package/src/modules/dashboards/api/widgets/data/batch/route.ts

@@ -9,6 +9,8 @@ import {
   createWidgetDataService,
   type WidgetDataRequest,
   WidgetDataValidationError,
+  WidgetDataScanLimitError,
+  WidgetDataEncryptionUnavailableError,
 } from '../../../../services/widgetDataService'
 import { runWidgetDataBatch } from '../../../../lib/widgetDataBatch'
 import type { AnalyticsRegistry } from '../../../../services/analyticsRegistry'
@@ -125,6 +127,8 @@ export async function POST(req: Request) {
       fetchOne: (request) => service.fetchWidgetData(request),
       describeError: (error) =>
         error instanceof WidgetDataValidationError
+        || error instanceof WidgetDataScanLimitError
+        || error instanceof WidgetDataEncryptionUnavailableError
           ? error.message
           : 'An error occurred while processing your request',
     })

package/src/modules/dashboards/api/widgets/data/route.ts

@@ -8,6 +8,8 @@ import {
   createWidgetDataService,
   type WidgetDataRequest,
   WidgetDataValidationError,
+  WidgetDataScanLimitError,
+  WidgetDataEncryptionUnavailableError,
 } from '../../../services/widgetDataService'
 import type { AnalyticsRegistry } from '../../../services/analyticsRegistry'
 import { runApiInterceptorsBefore } from '@open-mercato/shared/lib/crud/interceptor-runner'
@@ -129,6 +131,15 @@ export async function POST(req: Request) {
     if (err instanceof WidgetDataValidationError) {
       return NextResponse.json({ error: err.message }, { status: 400 })
     }
+    // The dataset is too large to group in application code; surface why rather than a masked 500.
+    if (err instanceof WidgetDataScanLimitError) {
+      return NextResponse.json({ error: err.message }, { status: 422 })
+    }
+    // Encryption is configured but currently unresolvable. Refusing the request keeps ciphertext and
+    // silently-wrong buckets out of the response, and 503 tells the client it is worth retrying.
+    if (err instanceof WidgetDataEncryptionUnavailableError) {
+      return NextResponse.json({ error: err.message }, { status: 503 })
+    }
     return NextResponse.json(
       { error: 'An error occurred while processing your request' },
       { status: 500 },
@@ -156,8 +167,18 @@ const widgetDataPostDoc: OpenApiMethodDoc = {
   errors: [
     { status: 400, description: 'Invalid request payload', schema: dashboardsErrorSchema },
     { status: 401, description: 'Authentication required', schema: dashboardsErrorSchema },
+    {
+      status: 422,
+      description: 'Too many rows to group an encrypted field in application code',
+      schema: dashboardsErrorSchema,
+    },
     { status: 403, description: 'Missing analytics.view feature', schema: dashboardsErrorSchema },
     { status: 500, description: 'Internal server error', schema: dashboardsErrorSchema },
+    {
+      status: 503,
+      description: 'Encryption is configured but the group source cannot currently be resolved',
+      schema: dashboardsErrorSchema,
+    },
   ],
 }
 

package/src/modules/dashboards/lib/aggregations.ts

@@ -97,54 +97,56 @@ export type BuildAggregationQueryOptions = {
   registry: AnalyticsRegistry
 }
 
-export function buildAggregationQuery(options: BuildAggregationQueryOptions): AggregationQuery | null {
-  const { registry } = options
-  const config = registry.getEntityTypeConfig(options.entityType)
-  if (!config) return null
-
-  const metricMapping = registry.getFieldMapping(options.entityType, options.metric.field)
-  if (!metricMapping) return null
-
-  const params: unknown[] = []
+export type ResolvedGroupExpression = {
+  expression: string
+  /** Column the group expression reads from — the encryption map is keyed by this. */
+  dbColumn: string
+  /** JSONB path below `dbColumn`, when the groupBy field used path notation. */
+  jsonPath: string | null
+}
 
-  const tableName = config.schema ? `"${config.schema}"."${config.tableName}"` : `"${config.tableName}"`
-  const aggregateExpr = buildAggregateExpression(options.metric.aggregate, metricMapping.dbColumn)
+/**
+ * Resolves the SQL expression a groupBy field maps to, together with the underlying column so
+ * callers can decide whether the source is encrypted at rest before grouping over it (#4622).
+ */
+export function resolveGroupExpression(
+  registry: AnalyticsRegistry,
+  entityType: string,
+  groupBy: { field: string; granularity?: DateGranularity },
+): ResolvedGroupExpression | null {
+  const groupMapping = registry.getFieldMapping(entityType, groupBy.field)
 
-  let selectClause = `SELECT ${aggregateExpr} AS value`
-  let groupByClause = ''
-  let orderByClause = ''
-  let limitClause = ''
-
-  if (options.groupBy) {
-    let groupMapping = registry.getFieldMapping(options.entityType, options.groupBy.field)
-    let groupExpr: string | null = null
-
-    // Handle JSONB path notation (e.g., shippingAddressSnapshot.region)
-    if (!groupMapping && options.groupBy.field.includes('.')) {
-      const [baseField, ...pathParts] = options.groupBy.field.split('.')
-      const baseMapping = registry.getFieldMapping(options.entityType, baseField)
-      if (baseMapping?.type === 'jsonb') {
-        groupExpr = buildJsonbFieldExpression(baseMapping.dbColumn, pathParts.join('.'))
-      }
-    } else if (groupMapping) {
-      if (groupMapping.type === 'timestamp' && options.groupBy.granularity) {
-        groupExpr = buildDateTruncExpression(groupMapping.dbColumn, options.groupBy.granularity)
-      } else {
-        groupExpr = groupMapping.dbColumn
-      }
+  // Handle JSONB path notation (e.g., shippingAddressSnapshot.region)
+  if (!groupMapping && groupBy.field.includes('.')) {
+    const [baseField, ...pathParts] = groupBy.field.split('.')
+    const baseMapping = registry.getFieldMapping(entityType, baseField)
+    if (baseMapping?.type !== 'jsonb') return null
+    const jsonPath = pathParts.join('.')
+    return {
+      expression: buildJsonbFieldExpression(baseMapping.dbColumn, jsonPath),
+      dbColumn: baseMapping.dbColumn,
+      jsonPath,
     }
+  }
 
-    if (groupExpr) {
-      selectClause = `SELECT ${groupExpr} AS group_key, ${aggregateExpr} AS value`
-      groupByClause = `GROUP BY ${groupExpr}`
-      orderByClause = `ORDER BY value DESC`
+  if (!groupMapping) return null
 
-      if (options.groupBy.limit && options.groupBy.limit > 0) {
-        limitClause = `LIMIT ${Math.min(options.groupBy.limit, 100)}`
-      }
+  if (groupMapping.type === 'timestamp' && groupBy.granularity) {
+    return {
+      expression: buildDateTruncExpression(groupMapping.dbColumn, groupBy.granularity),
+      dbColumn: groupMapping.dbColumn,
+      jsonPath: null,
     }
   }
 
+  return { expression: groupMapping.dbColumn, dbColumn: groupMapping.dbColumn, jsonPath: null }
+}
+
+function buildWhereClause(
+  options: Pick<BuildAggregationQueryOptions, 'entityType' | 'dateRange' | 'filters' | 'scope' | 'registry'>,
+): { clause: string; params: unknown[] } {
+  const { registry } = options
+  const params: unknown[] = []
   const whereClauses: string[] = []
 
   whereClauses.push(`tenant_id = ?`)
@@ -215,11 +217,88 @@ export function buildAggregationQuery(options: BuildAggregationQueryOptions): Ag
     }
   }
 
-  const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : ''
+  return { clause: whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '', params }
+}
+
+function resolveTableName(config: EntityTypeConfig): string {
+  return config.schema ? `"${config.schema}"."${config.tableName}"` : `"${config.tableName}"`
+}
+
+export function buildAggregationQuery(options: BuildAggregationQueryOptions): AggregationQuery | null {
+  const { registry } = options
+  const config = registry.getEntityTypeConfig(options.entityType)
+  if (!config) return null
+
+  const metricMapping = registry.getFieldMapping(options.entityType, options.metric.field)
+  if (!metricMapping) return null
+
+  const tableName = resolveTableName(config)
+  const aggregateExpr = buildAggregateExpression(options.metric.aggregate, metricMapping.dbColumn)
+
+  let selectClause = `SELECT ${aggregateExpr} AS value`
+  let groupByClause = ''
+  let orderByClause = ''
+  let limitClause = ''
+
+  if (options.groupBy) {
+    const resolved = resolveGroupExpression(registry, options.entityType, options.groupBy)
+
+    if (resolved) {
+      selectClause = `SELECT ${resolved.expression} AS group_key, ${aggregateExpr} AS value`
+      groupByClause = `GROUP BY ${resolved.expression}`
+      // NULLS LAST is stated explicitly (PostgreSQL defaults DESC to NULLS FIRST) so a group limit
+      // keeps the highest-value buckets instead of the empty ones, and so the application-side
+      // encrypted path can mirror the same ordering (#4622).
+      orderByClause = `ORDER BY value DESC NULLS LAST`
+
+      if (options.groupBy.limit && options.groupBy.limit > 0) {
+        limitClause = `LIMIT ${Math.min(options.groupBy.limit, 100)}`
+      }
+    }
+  }
+
+  const where = buildWhereClause(options)
+
+  const sql = [selectClause, `FROM ${tableName}`, where.clause, groupByClause, orderByClause, limitClause]
+    .filter(Boolean)
+    .join(' ')
+
+  return { sql, params: where.params }
+}
+
+export type BuildGroupSourceRowsQueryOptions = Omit<BuildAggregationQueryOptions, 'groupBy'> & {
+  /** Column holding the (encrypted) group source, resolved via `resolveGroupExpression`. */
+  groupColumn: string
+  /** Hard cap on scanned rows; callers fetch `rowLimit + 1` to detect overflow. */
+  rowLimit: number
+}
+
+/**
+ * Builds a per-row query for group sources that cannot be grouped in SQL because the column is
+ * encrypted at rest. Returns the raw group source alongside the metric value so the caller can
+ * decrypt, then aggregate in application code (#4622).
+ */
+export function buildGroupSourceRowsQuery(options: BuildGroupSourceRowsQueryOptions): AggregationQuery | null {
+  const { registry } = options
+  const config = registry.getEntityTypeConfig(options.entityType)
+  if (!config) return null
+
+  const metricMapping = registry.getFieldMapping(options.entityType, options.metric.field)
+  if (!metricMapping) return null
+
+  if (!isSafeIdentifier(options.groupColumn)) {
+    throw new Error(`Invalid group column: ${options.groupColumn}`)
+  }
 
-  const sql = [selectClause, `FROM ${tableName}`, whereClause, groupByClause, orderByClause, limitClause]
+  const where = buildWhereClause(options)
+  const sql = [
+    `SELECT ${options.groupColumn} AS group_source, ${metricMapping.dbColumn} AS metric_value`,
+    `FROM ${resolveTableName(config)}`,
+    where.clause,
+    `LIMIT ${Math.max(1, Math.floor(options.rowLimit)) + 1}`,
+  ]
     .filter(Boolean)
     .join(' ')
 
-  return { sql, params }
+  return { sql, params: where.params }
 }

package/src/modules/dashboards/lib/exactDecimal.ts

@@ -0,0 +1,96 @@
+/**
+ * Minimal exact decimal arithmetic for application-side aggregation of PostgreSQL `numeric`
+ * columns. The SQL path sums money in `numeric` and converts once, so the application path must
+ * not accumulate in binary floating point — `0.1 + 0.2` would drift the totals of encrypted group
+ * sources away from the plaintext ones (#4622).
+ *
+ * A value is stored as scaled integer `units` plus a decimal `scale`, so `12.34` is
+ * `{ units: 1234n, scale: 2 }`. Conversion to `number` happens only at the response boundary.
+ */
+
+export type ExactDecimal = {
+  units: bigint
+  scale: number
+}
+
+const DECIMAL_PATTERN = /^([+-]?)(\d*)(?:\.(\d*))?(?:[eE]([+-]?\d+))?$/
+
+export const EXACT_DECIMAL_ZERO: ExactDecimal = { units: 0n, scale: 0 }
+
+function pow10(exponent: number): bigint {
+  return 10n ** BigInt(exponent)
+}
+
+/**
+ * Parses a `numeric` column value. The PostgreSQL driver hands these over as strings to preserve
+ * precision, so the string form is authoritative; numbers and bigints are accepted for callers that
+ * already hold a JavaScript value. Returns `null` for anything that is not a finite decimal.
+ */
+export function parseExactDecimal(raw: unknown): ExactDecimal | null {
+  if (raw === null || raw === undefined) return null
+  if (typeof raw === 'bigint') return { units: raw, scale: 0 }
+
+  let text: string
+  if (typeof raw === 'number') {
+    if (!Number.isFinite(raw)) return null
+    text = String(raw)
+  } else if (typeof raw === 'string') {
+    text = raw.trim()
+  } else {
+    return null
+  }
+
+  if (!text) return null
+
+  const match = DECIMAL_PATTERN.exec(text)
+  if (!match) return null
+
+  const [, sign, intPart = '', fracPart = '', exponentPart] = match
+  if (!intPart && !fracPart) return null
+
+  const digits = `${intPart}${fracPart}`
+  let units = BigInt(digits)
+  let scale = fracPart.length
+
+  if (exponentPart) {
+    scale -= Number(exponentPart)
+    if (scale < 0) {
+      units *= pow10(-scale)
+      scale = 0
+    }
+  }
+
+  if (sign === '-') units = -units
+  return { units, scale }
+}
+
+function rescale(value: ExactDecimal, scale: number): bigint {
+  return scale === value.scale ? value.units : value.units * pow10(scale - value.scale)
+}
+
+export function addExactDecimal(left: ExactDecimal, right: ExactDecimal): ExactDecimal {
+  const scale = Math.max(left.scale, right.scale)
+  return { units: rescale(left, scale) + rescale(right, scale), scale }
+}
+
+export function compareExactDecimal(left: ExactDecimal, right: ExactDecimal): number {
+  const scale = Math.max(left.scale, right.scale)
+  const leftUnits = rescale(left, scale)
+  const rightUnits = rescale(right, scale)
+  if (leftUnits === rightUnits) return 0
+  return leftUnits < rightUnits ? -1 : 1
+}
+
+export function exactDecimalToString(value: ExactDecimal): string {
+  const negative = value.units < 0n
+  const digits = (negative ? -value.units : value.units).toString().padStart(value.scale + 1, '0')
+  const boundary = digits.length - value.scale
+  const integerPart = digits.slice(0, boundary)
+  const fractionPart = value.scale > 0 ? digits.slice(boundary) : ''
+  return `${negative ? '-' : ''}${integerPart}${fractionPart ? `.${fractionPart}` : ''}`
+}
+
+/** Response-boundary conversion: the only place where the exact value becomes a binary float. */
+export function exactDecimalToNumber(value: ExactDecimal): number {
+  return Number(exactDecimalToString(value))
+}