@open-mercato/core 0.6.8-develop.6924.1.a8d208fcdc → 0.6.8-develop.6930.1.1e5976efc3

package/src/modules/query_index/lib/engine.ts

@@ -38,6 +38,11 @@ import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
 
 const logger = createLogger('query_index').child({ component: 'engine' })
 
+/** Operators `buildCfFilterExpression` compiles; anything else yields no predicate. */
+const CF_FILTER_SUPPORTED_OPS = new Set<FilterOp>([
+  'eq', 'ne', 'in', 'nin', 'like', 'ilike', 'exists', 'gt', 'gte', 'lt', 'lte',
+])
+
 const DECRYPT_CONCURRENCY = 8
 const AUTO_REINDEX_DEBOUNCE_DEFAULT_MS = 30_000
 const AUTO_REINDEX_DEBOUNCE_MAX_SCOPES = 10_000
@@ -742,9 +747,14 @@ export class HybridQueryEngine implements QueryEngine {
         return next
       }
 
+      // Custom-field leaves that belong to an OR disjunct are handled together with
+      // the base ones below; applying them here would AND them onto every disjunct.
+      const regularCfFilters = cfFilters.filter((filter) => !filter.orGroup)
+      const orGroupCfFilters = cfFilters.filter((filter) => filter.orGroup)
+
       const applyCfFilters = (q: AnyBuilder): AnyBuilder => {
         let next = q
-        for (const filter of cfFilters) {
+        for (const filter of regularCfFilters) {
           next = this.applyCfFilterAcrossSources(
             next, filter.field, filter.op, filter.value, indexSources, searchRuntime,
           )
@@ -775,28 +785,60 @@ export class HybridQueryEngine implements QueryEngine {
         return next
       }
 
+      // Also used by the optimized count path, which builds from a bare base table with
+      // no index/custom-field joins. Emitting a cf predicate there would reference an
+      // alias that query has never joined, so this stays safe only while
+      // `canOptimizeCount` is false whenever any cf filter exists (see `hasCustomFieldFilters`).
       const applyOrGroupedBaseFilters = (q: AnyBuilder): AnyBuilder => {
-        if (orGroupFilters.length === 0) return q
-        const groups = new Map<string, BaseFilter[]>()
+        if (orGroupFilters.length === 0 && orGroupCfFilters.length === 0) return q
+        // `BaseFilter` here is just the normalized-leaf shape; the `cf` bucket holds
+        // custom-field leaves, which never resolve to a base column.
+        const groups = new Map<string, { base: BaseFilter[]; cf: BaseFilter[] }>()
+        const bucketFor = (orGroup: string) => {
+          const existing = groups.get(orGroup) ?? { base: [], cf: [] }
+          groups.set(orGroup, existing)
+          return existing
+        }
         for (const filter of orGroupFilters) {
           if (!filter.orGroup) continue
-          const existing = groups.get(filter.orGroup) ?? []
-          existing.push(filter)
-          groups.set(filter.orGroup, existing)
+          bucketFor(filter.orGroup).base.push(filter)
+        }
+        // Custom-field leaves belong to the same disjunct as the base ones sharing
+        // their orGroup, so a mixed `base OR cf` filter unites rather than intersects.
+        for (const filter of orGroupCfFilters) {
+          if (!filter.orGroup) continue
+          bucketFor(filter.orGroup).cf.push(filter)
         }
-        const groupList = Array.from(groups.values()).filter((g) => g.length > 0)
-        if (groupList.length === 0) return q
+        // A cf leaf that compiles to no predicate is dropped, and a disjunct left with
+        // no leaves is dropped too — an empty AND would read as TRUE and widen the result.
+        const applicableGroups = Array.from(groups.values())
+          .map((group) => ({
+            base: group.base,
+            cf: group.cf.filter((filter) =>
+              this.cfFilterHasPredicate(filter.op, filter.value, indexSources, searchRuntime),
+            ),
+          }))
+          .filter((group) => group.base.length > 0 || group.cf.length > 0)
+        if (applicableGroups.length === 0) return q
         // Combine all groups in a single WHERE: disjuncts are OR'd together; within
         // each disjunct, fields are AND'd. Building this as separate `.where()` calls
         // would AND the disjuncts (wrong semantics).
-        return q.where((eb: any) => eb.or(
-          groupList.map((groupFilters) => {
-            const parts = groupFilters.map((filter) =>
-              this.buildBaseFilterExpression(eb, filter, resolveBaseColumn, qualify, entity, searchRuntime),
-            )
+        return q.where((eb: any) => {
+          const disjuncts = applicableGroups.map((groupFilters) => {
+            const parts = [
+              ...groupFilters.base.map((filter) =>
+                this.buildBaseFilterExpression(eb, filter, resolveBaseColumn, qualify, entity, searchRuntime),
+              ),
+              ...groupFilters.cf.map((filter) =>
+                this.buildCfFilterExpression(
+                  eb, filter.field, filter.op, filter.value, indexSources, searchRuntime,
+                ),
+              ),
+            ]
             return parts.length === 1 ? parts[0] : eb.and(parts)
-          }),
-        ))
+          })
+          return disjuncts.length === 1 ? disjuncts[0] : eb.or(disjuncts)
+        })
       }
 
       const applyAliasScopes = async (target: AnyBuilder, aliasName: string): Promise<AnyBuilder> => {
@@ -1350,37 +1392,44 @@ export class HybridQueryEngine implements QueryEngine {
     return sql<string | null>`coalesce(${sql.join(parts, sql`, `)})`
   }
 
-  private applyCfFilterAcrossSources(
-    builder: AnyBuilder,
+  /**
+   * Build a custom-field predicate as a standalone expression.
+   *
+   * Returning an expression rather than mutating a builder is what lets OR-grouped
+   * cf leaves be combined with the base-column ones inside a single `WHERE`
+   * (#5039). `null` means "no predicate" — an empty search-token set or an
+   * unsupported operator — and callers must treat that as the leaf being absent.
+   */
+  private buildCfFilterExpression(
+    eb: any,
     key: string,
     op: FilterOp,
     value: unknown,
     sources: IndexDocSource[],
     search?: SearchRuntime
-  ): AnyBuilder {
-    if (!sources.length) return builder
+  ): any | null {
+    if (!sources.length) return null
     if ((op === 'like' || op === 'ilike') && search?.enabled && typeof value === 'string') {
       const tokens = tokenizeText(String(value), search.config)
       const hashes = tokens.hashes
-      if (hashes.length) {
-        const applied = this.applyMultiSourceSearchExists(builder, sources, key, hashes, search)
-        this.logSearchDebug('search:cf-filter-across', {
-          entity: sources.map((src) => src.entityId),
-          field: key, tokens: tokens.tokens, hashes, applied,
-          tenantId: search.tenantId ?? null, organizationScope: search.organizationScope,
-        })
-        if (applied.builder !== builder) return applied.builder
-      } else {
+      if (!hashes.length) {
         this.logSearchDebug('search:cf-skip-empty-hashes', {
           entity: sources.map((src) => src.entityId), field: key, value,
         })
+        return null
       }
-      return builder
+      const expression = this.buildMultiSourceSearchExists(eb, sources, key, hashes, search)
+      this.logSearchDebug('search:cf-filter-across', {
+        entity: sources.map((src) => src.entityId),
+        field: key, tokens: tokens.tokens, hashes, applied: expression !== null,
+        tenantId: search.tenantId ?? null, organizationScope: search.organizationScope,
+      })
+      return expression
     }
 
     const textExpr = this.buildCfTextExprSql(key, sources)
     const jsonExpr = this.buildCfJsonExprSql(key, sources)
-    if (!textExpr || !jsonExpr) return builder
+    if (!textExpr || !jsonExpr) return null
 
     const arrContains = (val: unknown) => sql<boolean>`${jsonExpr} @> ${JSON.stringify([val])}::jsonb`
 
@@ -1389,58 +1438,98 @@ export class HybridQueryEngine implements QueryEngine {
         // An unset custom field has no array element to contain, so the
         // arrContains branch cannot match it — compare the text value only.
         return value === null
-          ? builder.where(sql<boolean>`${textExpr} is null`)
-          : builder.where((eb: any) => eb.or([
+          ? sql<boolean>`${textExpr} is null`
+          : eb.or([
               sql<boolean>`${textExpr} = ${value}`,
               arrContains(value),
-            ]))
+            ])
       case 'ne':
         return value === null
-          ? builder.where(sql<boolean>`${textExpr} is not null`)
-          : builder.where(sql<boolean>`${textExpr} <> ${value}`)
+          ? sql<boolean>`${textExpr} is not null`
+          : sql<boolean>`${textExpr} <> ${value}`
       case 'in': {
         const values = this.toArray(value)
-        return builder.where((eb: any) => eb.or(
+        return eb.or(
           values.flatMap((val) => [
             sql<boolean>`${textExpr} = ${val}`,
             arrContains(val),
           ])
-        ))
+        )
       }
       case 'nin': {
         const values = this.toArray(value)
-        return builder.where(sql<boolean>`${textExpr} not in (${sql.join(values.map((v) => sql`${v}`), sql`, `)})`)
+        return sql<boolean>`${textExpr} not in (${sql.join(values.map((v) => sql`${v}`), sql`, `)})`
       }
       case 'like':
-        return builder.where(sql<boolean>`${textExpr} like ${value}`)
+        return sql<boolean>`${textExpr} like ${value}`
       case 'ilike':
-        return builder.where(sql<boolean>`${textExpr} ilike ${value}`)
+        return sql<boolean>`${textExpr} ilike ${value}`
       case 'exists':
         return value
-          ? builder.where(sql<boolean>`${textExpr} is not null`)
-          : builder.where(sql<boolean>`${textExpr} is null`)
+          ? sql<boolean>`${textExpr} is not null`
+          : sql<boolean>`${textExpr} is null`
       case 'gt':
       case 'gte':
       case 'lt':
       case 'lte': {
         const operator = sql.raw(op === 'gt' ? '>' : op === 'gte' ? '>=' : op === 'lt' ? '<' : '<=')
-        return builder.where(sql<boolean>`${textExpr} ${operator} ${value}`)
+        return sql<boolean>`${textExpr} ${operator} ${value}`
       }
       default:
-        return builder
+        return null
     }
   }
 
-  /** Apply a search-token EXISTS subquery across multiple sources (OR-joined). */
-  private applyMultiSourceSearchExists(
+  /**
+   * Whether `buildCfFilterExpression` will produce a predicate for this leaf.
+   *
+   * Decidable without an ExpressionBuilder, which lets callers skip the `.where()`
+   * entirely (rather than emitting a tautology) and lets the OR-group builder drop
+   * leaves before it has to decide whether a disjunct survives.
+   */
+  private cfFilterHasPredicate(
+    op: FilterOp,
+    value: unknown,
+    sources: IndexDocSource[],
+    search?: SearchRuntime,
+  ): boolean {
+    if (!sources.length) return false
+    if ((op === 'like' || op === 'ilike') && search?.enabled && typeof value === 'string') {
+      return tokenizeText(String(value), search.config).hashes.length > 0
+    }
+    return CF_FILTER_SUPPORTED_OPS.has(op)
+  }
+
+  private applyCfFilterAcrossSources(
     builder: AnyBuilder,
+    key: string,
+    op: FilterOp,
+    value: unknown,
+    sources: IndexDocSource[],
+    search?: SearchRuntime
+  ): AnyBuilder {
+    if (!this.cfFilterHasPredicate(op, value, sources, search)) {
+      // Preserve the pre-existing behaviour of dropping a leaf we cannot compile.
+      if (sources.length && (op === 'like' || op === 'ilike') && search?.enabled && typeof value === 'string') {
+        this.logSearchDebug('search:cf-skip-empty-hashes', {
+          entity: sources.map((src) => src.entityId), field: key, value,
+        })
+      }
+      return builder
+    }
+    return builder.where((eb: any) => this.buildCfFilterExpression(eb, key, op, value, sources, search))
+  }
+
+  /** Build a search-token EXISTS predicate across multiple sources (OR-joined). */
+  private buildMultiSourceSearchExists(
+    eb: any,
     sources: IndexDocSource[],
     key: string,
     hashes: string[],
     search: SearchRuntime,
-  ): { builder: AnyBuilder; applied: boolean } {
-    if (!sources.length || !hashes.length) return { builder, applied: false }
-    const next = builder.where((eb: any) => eb.or(
+  ): any | null {
+    if (!sources.length || !hashes.length) return null
+    return eb.or(
       sources.map((source) =>
         eb.exists(this.buildSearchTokensSub(eb, {
           entity: String(source.entityId),
@@ -1451,8 +1540,7 @@ export class HybridQueryEngine implements QueryEngine {
           mintAlias: search.mintAlias,
         }))
       )
-    ))
-    return { builder: next, applied: true }
+    )
   }
 
   /** Construct a search-token EXISTS subquery using the given ExpressionBuilder. */

package/src/modules/sales/api/documents/factory.ts

@@ -22,6 +22,7 @@ import { parseScopedCommandInput, resolveCrudRecordId } from '../utils'
 import { documentUpdateSchema } from '../../commands/documents'
 import { buildIlikeTerm } from '@open-mercato/shared/lib/db/buildIlikeTerm'
 import { parseBooleanToken } from '@open-mercato/shared/lib/boolean'
+import { parseIdsParam } from '@open-mercato/shared/lib/crud/ids'
 import type { RbacService } from '@open-mercato/core/modules/auth/services/rbacService'
 import { recalculateOrderTotalsForDisplay } from '../../commands/returns'
 import { parseDecryptedFieldValue } from '@open-mercato/shared/lib/encryption/tenantDataEncryptionService'
@@ -89,6 +90,18 @@ const listSchema = z
     id: z.string().uuid().optional(),
     customerId: z.string().uuid().optional(),
     channelId: z.string().uuid().optional(),
+    channelIds: z
+      .string()
+      .optional()
+      .describe(
+        'Comma-separated sales channel uuids; matches documents on any of them. Capped at 200 ids, malformed entries are dropped. Ignored when channelId is supplied; combines with channelIdsEmpty.',
+      ),
+    channelIdsEmpty: z
+      .string()
+      .optional()
+      .describe(
+        'Boolean token; matches documents with no sales channel. Ignored when channelId is supplied; combines with channelIds.',
+      ),
     lineItemCountMin: z.coerce.number().min(0).optional(),
     lineItemCountMax: z.coerce.number().min(0).optional(),
     totalNetMin: z.coerce.number().optional(),
@@ -117,8 +130,24 @@ function buildFilters(query: ListQuery, numberColumn: string, kind: DocumentKind
   if (query.customerId) {
     filters.customer_entity_id = { $eq: query.customerId }
   }
+  // Singular wins over plural, mirroring how `api/channels/route.ts` resolves `id` before `ids`.
+  // An all-malformed `channelIds` narrows to no channel filter rather than an empty-set filter, so
+  // a typo returns the unfiltered page instead of silently returning zero rows.
   if (query.channelId) {
     filters.channel_id = { $eq: query.channelId }
+  } else {
+    const channelIds = parseIdsParam(query.channelIds)
+    const wantsUnassigned = parseBooleanToken(query.channelIdsEmpty) === true
+    if (channelIds.length && wantsUnassigned) {
+      // A channel multi-select with an "(No channel)" entry produces both at once, so they combine
+      // rather than one silently dropping the other. `filters.$or` is a single key — a future filter
+      // that also needs `$or` would clobber this one; nothing else in this factory writes it today.
+      filters.$or = [{ channel_id: { $in: channelIds } }, { channel_id: { $exists: false } }]
+    } else if (wantsUnassigned) {
+      filters.channel_id = { $exists: false }
+    } else if (channelIds.length) {
+      filters.channel_id = { $in: channelIds }
+    }
   }
   const lineRange: Record<string, number> = {}
   if (typeof query.lineItemCountMin === 'number') lineRange.$gte = query.lineItemCountMin

package/src/modules/sales/commands/documents.ts

@@ -3632,6 +3632,22 @@ function normalizeTagIds(tags?: Array<string | null | undefined>): string[] {
   return Array.from(set);
 }
 
+// A loaded assignment exposes its tag as an entity, an unwrapped id or the raw
+// column depending on how it was fetched; the snapshot loaders read it the same
+// defensive way. Returns null when the id cannot be determined, which callers
+// must treat as "unrecognised" rather than as a match.
+function readAssignmentTagId(assignment: {
+  tag?: unknown;
+}): string | null {
+  const tag = (assignment as { tag?: unknown }).tag;
+  if (typeof tag === "string") return tag;
+  return (
+    (tag as { id?: string } | null | undefined)?.id ??
+    (assignment as { tag_id?: string }).tag_id ??
+    null
+  );
+}
+
 function buildTagChange(
   beforeTags: TagAssignmentSnapshot[] | undefined,
   afterTags: TagAssignmentSnapshot[] | undefined,
@@ -3719,6 +3735,28 @@ function buildDocumentUpdateChangeKeys(kind: SalesDocumentKind, input: DocumentU
   return Array.from(keys);
 }
 
+// Writes only the difference between the stored and the incoming tag set, and
+// no-ops when they are equal. The previous delete-all-then-reinsert rewrote an
+// unchanged set on every save that carried `tags`: on a large
+// sales_document_tag_assignments that delete was the dominant cost of a bulk
+// import, and it also reset each assignment's id/created_at, churned the unique
+// index and left two dead tuples per document per save.
+//
+// Semantics are unchanged — a tag removed out of band still differs from the
+// incoming set and is still restored. Removals go through the UnitOfWork rather
+// than nativeDelete because the assignments are loaded (managed) here; a native
+// delete would leave stale identities behind. The add and remove sets are
+// disjoint by construction, so MikroORM's insert-before-delete commit order can
+// never transiently violate the unique (tag_id, document_id, document_kind).
+//
+// The assignment read below is deliberately scoped by document only, with no
+// organization/tenant predicate — unlike the display reads in
+// api/documents/factory.ts, and unlike the tag-scope check above. It has to
+// match the scope of the unique constraint it reconciles against, which spans
+// (tag_id, document_id, document_kind) and no scope columns. Narrowing the read
+// would hide a mis-scoped row from the diff, put its tag in the add set, and
+// turn the insert into a unique-constraint violation. The document itself is
+// already scope-checked by the caller before this runs.
 async function syncSalesDocumentTags(
   em: EntityManager,
   params: {
@@ -3731,29 +3769,38 @@ async function syncSalesDocumentTags(
 ) {
   if (params.tagIds === undefined) return;
   const tagIds = normalizeTagIds(params.tagIds);
-  if (tagIds.length === 0) {
-    await em.nativeDelete(SalesDocumentTagAssignment, {
-      documentId: params.documentId,
-      documentKind: params.kind,
-    });
-    return;
-  }
-  const tagsInScope = await em.find(SalesDocumentTag, {
-    id: { $in: tagIds },
-    organizationId: params.organizationId,
-    tenantId: params.tenantId,
-  });
-  if (tagsInScope.length !== tagIds.length) {
-    throw new CrudHttpError(400, {
-      error: "One or more tags not found for this scope",
+  const byId = new Map<string, SalesDocumentTag>();
+  if (tagIds.length > 0) {
+    const tagsInScope = await em.find(SalesDocumentTag, {
+      id: { $in: tagIds },
+      organizationId: params.organizationId,
+      tenantId: params.tenantId,
     });
+    if (tagsInScope.length !== tagIds.length) {
+      throw new CrudHttpError(400, {
+        error: "One or more tags not found for this scope",
+      });
+    }
+    tagsInScope.forEach((tag) => byId.set(tag.id, tag));
   }
-  const byId = new Map(tagsInScope.map((tag) => [tag.id, tag]));
-  await em.nativeDelete(SalesDocumentTagAssignment, {
+  const existing = await em.find(SalesDocumentTagAssignment, {
     documentId: params.documentId,
     documentKind: params.kind,
   });
-  for (const tagId of tagIds) {
+  const existingByTagId = new Map<string, (typeof existing)[number]>();
+  existing.forEach((assignment) => {
+    const tagId = readAssignmentTagId(assignment);
+    if (tagId) existingByTagId.set(tagId, assignment);
+  });
+  const wanted = new Set(tagIds);
+  const toRemove = existing.filter((assignment) => {
+    const tagId = readAssignmentTagId(assignment);
+    return tagId === null || !wanted.has(tagId);
+  });
+  const toAdd = tagIds.filter((tagId) => !existingByTagId.has(tagId));
+  if (toRemove.length === 0 && toAdd.length === 0) return;
+  toRemove.forEach((assignment) => em.remove(assignment));
+  for (const tagId of toAdd) {
     const tag = byId.get(tagId);
     if (!tag) continue;
     const assignment = em.create(SalesDocumentTagAssignment, {

package/src/modules/sales/data/entities.ts

@@ -1947,6 +1947,7 @@ export class SalesDocumentTag {
 
 @Entity({ tableName: 'sales_document_tag_assignments' })
 @Index({ name: 'sales_document_tag_assignments_scope_idx', properties: ['organizationId', 'tenantId'] })
+@Index({ name: 'sales_document_tag_assignments_document_idx', properties: ['documentId'] })
 @Unique({
   name: 'sales_document_tag_assignments_unique',
   properties: ['tag', 'documentId', 'documentKind'],

package/src/modules/sales/migrations/.snapshot-open-mercato.json

@@ -2341,6 +2341,16 @@
           "primary": false,
           "unique": false
         },
+        {
+          "columnNames": [
+            "document_id"
+          ],
+          "composite": false,
+          "constraint": false,
+          "keyName": "sales_document_tag_assignments_document_idx",
+          "primary": false,
+          "unique": false
+        },
         {
           "columnNames": [
             "tag_id",

package/src/modules/sales/migrations/Migration20260806120000_sales_tag_assignment_document_idx.ts

@@ -0,0 +1,59 @@
+import { Migration } from '@mikro-orm/migrations';
+
+// sales_document_tag_assignments had no index whose leading column is
+// document_id: the table carries PK(id), (organization_id, tenant_id) and the
+// unique (tag_id, document_id, document_kind) — and the unique index leads with
+// tag_id, which a document-scoped predicate never supplies. Every read and
+// write that resolves a document's tags therefore degraded to a sequential scan
+// as the table grew: syncSalesDocumentTags' per-save delete
+// (commands/documents.ts) and the order/quote grid's
+// `left join ... on id = document_id` (api/documents/factory.ts) alike. On a
+// 2.85M-row / 868MB production table the delete measured 139.9 ms — a full
+// table scan to remove the two rows belonging to one order.
+//
+// document_kind is deliberately omitted. sales_orders.id and sales_quotes.id
+// are both gen_random_uuid() from their own tables, so a document_id already
+// determines its kind; the second column adds no selectivity for ~30% more
+// index and document_kind is simply rechecked over the rows the seek returns.
+// The unique (tag_id, document_id, document_kind) index stays as-is — its
+// leading tag_id is load-bearing for sales.tags.delete (commands/tags.ts), for
+// the grid's tag filter and for the foreign-key check. Both column orders are
+// needed; two indexes is correct.
+//
+// The table is high-churn, so the index is built CONCURRENTLY to avoid blocking
+// writes during the build. CREATE INDEX CONCURRENTLY cannot run inside a
+// transaction, hence isTransactional() => false; the migration runner applies
+// migrations one-by-one, so this opt-out is safe. Drop first so retrying a
+// failed concurrent build removes PostgreSQL's invalid index stub instead of
+// letting IF NOT EXISTS silently accept it: a half-built concurrent index is
+// left INVALID, the planner ignores it, and IF NOT EXISTS would report success
+// while the table stays unindexed.
+//
+// Both halves follow Migration20260731105052_query_index, which builds
+// concurrently and drops first for the same reason. The earlier
+// Migration20260611103000_query_index builds concurrently but uses IF NOT
+// EXISTS without the drop — cited here only so a reader comparing the two knows
+// the divergence is deliberate rather than an oversight.
+//
+// On a database that does not have the index yet — every fresh install — the
+// drop is a no-op and there is no window without it. An operator who already
+// created this index out of band should expect it to be rebuilt here, and the
+// document-scoped scans to fall back to their pre-index cost until the
+// concurrent build finishes; on a large table, run this while the writers that
+// depend on it are paused.
+export class Migration20260806120000_sales_tag_assignment_document_idx extends Migration {
+
+  override isTransactional(): boolean {
+    return false;
+  }
+
+  override up(): void | Promise<void> {
+    this.addSql(`drop index concurrently if exists "sales_document_tag_assignments_document_idx";`);
+    this.addSql(`create index concurrently "sales_document_tag_assignments_document_idx" on "sales_document_tag_assignments" ("document_id");`);
+  }
+
+  override down(): void | Promise<void> {
+    this.addSql(`drop index concurrently if exists "sales_document_tag_assignments_document_idx";`);
+  }
+
+}