json-rest-api 1.0.29 → 2.0.0

package/plugins/core/lib/querying/knex-query-helpers.js

@@ -1,5 +1,118 @@
-import { analyzeRequiredIndexes, buildJoinChain } from './knex-cross-table-search.js'
+import { assertScalarQueryField } from '../querying-writing/field-utils.js'
+import { buildJoinChain } from './knex-cross-table-search.js'
 import { createStorageAdapterUtilities } from './storage-adapter-utils.js'
+import { unwrapQueryBuilderState } from './query-builder-utils.js'
+import { RestApiResourceError, RestApiValidationError } from '../../../../lib/rest-api-errors.js'
+import { resolveSortField } from './query-field-sort-helpers.js'
+import { databaseIdentityExpression } from '../querying-writing/database-value-normalizers.js'
+
+const referenceVisibility = Symbol('referenceVisibility')
+
+async function buildSearchJoinQuery ({ scopeName, scopes, context, adapterUtils, db }) {
+  const scope = scopes[scopeName]
+  const storageAdapter = adapterUtils.fetchStorageAdapter(scopeName)
+  if (!scope || !storageAdapter) throw new RestApiResourceError(`Related resource '${scopeName}' not found`, { subtype: 'related_type_not_found' })
+  const tableName = storageAdapter.getTableName()
+  const query = storageAdapter.buildBaseQuery({ transaction: db }).select(`${tableName}.*`)
+  const state = await scope.applyQueryFilters({
+    query,
+    scopeName,
+    tableName,
+    db,
+    storageAdapter,
+    schemaInfo: scope.vars.schemaInfo,
+    queryPurpose: 'search-join',
+    isAnyApi: storageAdapter.isCanonical()
+  }, { ...context, queryParams: {}, scopeName, schemaInfo: scope.vars.schemaInfo, storageAdapter })
+  return { query: unwrapQueryBuilderState(state, query) }
+}
+
+async function resolveSearchColumns (references, { scopes, context, adapterUtils, db, operation = 'filter' }) {
+  const columns = new Map()
+  for (const [key, { scopeName, field, alias }] of references) {
+    const column = adapterUtils.translateColumn(scopeName, field, alias)
+    const schemaInfo = scopes[scopeName]?.vars?.schemaInfo
+    const fieldDef = schemaInfo?.schemaStructure?.[field]
+    assertScalarQueryField(fieldDef, field, operation)
+    let idField, typeField, targets
+    if (fieldDef?.belongsTo) {
+      idField = field
+      targets = [fieldDef.belongsTo]
+    } else {
+      for (const [name, definition] of Object.entries({ ...schemaInfo?.schemaStructure, ...schemaInfo?.schemaRelationships })) {
+        const polymorphic = definition?.belongsToPolymorphic
+        if (!polymorphic) continue
+        const referenceId = polymorphic.idField || `${name}_id`
+        const referenceType = polymorphic.typeField || `${name}_type`
+        if (field !== referenceId && field !== referenceType) continue
+        idField = referenceId
+        typeField = referenceType
+        targets = polymorphic.types
+        break
+      }
+    }
+    if (!targets) {
+      columns.set(key, column)
+      continue
+    }
+
+    const selections = []
+    const targetAlias = alias === '__jra_reference' ? '__jra_reference_target' : '__jra_reference'
+    for (const target of targets) {
+      const state = await buildSearchJoinQuery({ scopeName: target, scopes, context, adapterUtils, db })
+      const selection = db.queryBuilder().from(state.query.as(targetAlias)).select(db.raw('1'))
+        .whereColumn(adapterUtils.translateColumn(target, 'id', targetAlias), adapterUtils.translateColumn(scopeName, idField, alias))
+      if (typeField) selection.where(adapterUtils.translateColumn(scopeName, typeField, alias), adapterUtils.translateFilterValue(scopeName, typeField, target))
+      selections.push(selection)
+    }
+    columns.set(key, { column, [referenceVisibility]: selections })
+  }
+  return columns
+}
+
+export async function prepareReferenceSortColumns ({ query, fields, scopeName, tableAlias, context }, dependencies) {
+  const { scopes, knex, getStorageAdapter } = dependencies
+  const storageAdapter = getStorageAdapter(scopeName)
+  const tableName = storageAdapter.getTableName()
+  const db = context.db || context.transaction || knex
+  const schemaInfo = scopes[scopeName].vars.schemaInfo
+  const adapterUtils = createStorageAdapterUtilities({
+    context: {
+      ...context, knexQuery: { scopeName, tableName, storageAdapter }
+    }
+  }, dependencies)
+  const references = new Map(fields.map(field => [field, {
+    scopeName, alias: tableName, field: resolveSortField(field, schemaInfo)
+  }]))
+  const resolved = await resolveSearchColumns(references, { scopes, context, adapterUtils, db, operation: 'sort' })
+  const columns = new Map()
+  if (![...resolved.values()].some(reference => reference?.[referenceVisibility])) return columns
+  const resultNames = new Set([
+    ...fields,
+    ...Object.keys(scopes[scopeName].vars.schemaInfo?.queryFields || {}),
+    ...Object.keys(schemaInfo.schemaStructure).flatMap(field => [field, storageAdapter.translateColumn(field)])
+  ])
+  let resultIndex = 0
+  for (const [field, reference] of resolved) {
+    const selections = reference?.[referenceVisibility]
+    if (!selections) continue
+    const joinAlias = `__jra_sort_${columns.size}${tableAlias === `__jra_sort_${columns.size}` ? '_target' : ''}`
+    while (resultNames.has(`__jra_sort_value_${resultIndex}`)) resultIndex++
+    const resultColumn = `__jra_sort_value_${resultIndex++}`
+    // A filtered copy preserves the source column's affinity and collation.
+    const visibleSource = storageAdapter.buildBaseQuery({ transaction: db })
+      .select({ __jra_id: `${tableName}.${storageAdapter.getIdColumn()}`, __jra_value: reference.column })
+      .where(function () {
+        if (selections.length === 0) this.whereRaw('1 = 0')
+        for (const selection of selections) this.orWhereExists(selection)
+      })
+    const column = `${joinAlias}.__jra_value`
+    query.leftJoin(visibleSource.as(joinAlias), `${tableAlias}.${storageAdapter.getIdColumn()}`, `${joinAlias}.__jra_id`)
+    query.select({ [resultColumn]: databaseIdentityExpression(db, column) })
+    columns.set(field, { column, resultColumn, referenceField: references.get(field).field })
+  }
+  return columns
+}
 
 // Resolve operator with sensible defaults for fields declared in searchSchema.
 // - If filterOperator is provided, use it as-is
@@ -10,22 +123,41 @@ export function resolveSearchOperator (fieldDef) {
 }
 
 // Apply a comparison for a single field/operator/value onto a query builder.
-// Handles contains/startsWith/endsWith, IN, BETWEEN, =/== null semantics,
-// and uses ILIKE for case-insensitive matching on Postgres.
+// Handles text matching, IN, BETWEEN, and explicit null comparisons.
 export function applyWhereForOperator ({ builder, columnRef, operator, value, knex, or = false }) {
   const method = or ? 'orWhere' : 'where'
   const methodNull = or ? 'orWhereNull' : 'whereNull'
   const methodIn = or ? 'orWhereIn' : 'whereIn'
+
+  const selections = columnRef?.[referenceVisibility]
+  if (selections) {
+    const visible = function () {
+      if (selections.length === 0) this.whereRaw('1 = 0')
+      for (const selection of selections) this.orWhereExists(selection)
+    }
+    // Compare the column itself to preserve driver affinity and collation.
+    builder[method](function () {
+      this.where(function () {
+        this.where(visible)
+        applyWhereForOperator({ builder: this, columnRef: columnRef.column, operator, value, knex })
+      }).orWhere(function () {
+        this.whereNot(visible)
+        applyWhereForOperator({ builder: this, columnRef: knex.raw('null'), operator, value, knex })
+      })
+    })
+    return
+  }
   const likeOp = 'like'
 
   const op = typeof operator === 'string' ? operator.toLowerCase() : operator
+  const textSearch = ['like', 'contains', 'startswith', 'endswith'].includes(op)
 
   // Normalize scalar from possibly array input
   const firstVal = Array.isArray(value) ? value[0] : value
 
   // Null handling for equality and string ops
   if (firstVal === null || firstVal === undefined) {
-    if (op === 'like' || op === 'contains' || op === 'startswith' || op === 'endswith') {
+    if (textSearch) {
       builder[methodNull](columnRef)
       return
     }
@@ -33,19 +165,18 @@ export function applyWhereForOperator ({ builder, columnRef, operator, value, kn
       builder[methodNull](columnRef)
       return
     }
+    if (op === '!=' || op === '<>' || op === 'is not') {
+      builder[or ? 'orWhereNotNull' : 'whereNotNull'](columnRef)
+      return
+    }
   }
 
   // Text search operators
-  if (op === 'like' || op === 'contains') {
-    builder[method](columnRef, likeOp, `%${String(firstVal)}%`)
-    return
-  }
-  if (op === 'startswith') {
-    builder[method](columnRef, likeOp, `${String(firstVal)}%`)
-    return
-  }
-  if (op === 'endswith') {
-    builder[method](columnRef, likeOp, `%${String(firstVal)}`)
+  if (textSearch) {
+    const textColumn = knex.client.config.client === 'pg' ? knex.raw('cast(?? as text)', [columnRef]) : columnRef
+    const prefix = op === 'startswith' ? '' : '%'
+    const suffix = op === 'endswith' ? '' : '%'
+    builder[method](textColumn, likeOp, `${prefix}${String(firstVal)}${suffix}`)
     return
   }
 
@@ -56,16 +187,10 @@ export function applyWhereForOperator ({ builder, columnRef, operator, value, kn
     return
   }
   if (op === 'between') {
-    const values = Array.isArray(value) ? value : [value]
-    if (values.length === 2) {
-      builder.whereBetween(columnRef, values)
-    } else if (values.length === 1) {
-      if (values[0] === null || values[0] === undefined) {
-        builder[methodNull](columnRef)
-      } else {
-        builder[method](columnRef, '=', values[0])
-      }
+    if (!Array.isArray(value) || value.length !== 2 || value.some(bound => bound == null)) {
+      throw new RestApiValidationError('A between filter requires exactly two non-null bounds')
     }
+    builder[or ? 'orWhereBetween' : 'whereBetween'](columnRef, value)
     return
   }
 
@@ -78,73 +203,6 @@ export function applyWhereForOperator ({ builder, columnRef, operator, value, kn
  *
  * @param {Object} hookParams - Hook parameters containing context
  * @param {Object} dependencies - Dependencies injected by the plugin
- *
- * @example
- * // Input: Search schema with polymorphic filter
- * const searchSchema = {
- *   commentable_title: {
- *     type: 'string',
- *     polymorphicField: 'commentable',  // Points to the polymorphic relationship
- *     targetFields: {
- *       posts: 'title',      // When commentable_type='posts', search posts.title
- *       videos: 'title',     // When commentable_type='videos', search videos.title
- *       articles: 'headline' // When commentable_type='articles', search articles.headline
- *     },
- *     filterOperator: 'like'
- *   }
- * };
- *
- * // Filter request: { commentable_title: 'JavaScript' }
- *
- * // Result: Adds conditional LEFT JOINs and WHERE conditions
- * // SQL generated:
- * // LEFT JOIN posts ON comments.commentable_type = 'posts' AND comments.commentable_id = posts.id
- * // LEFT JOIN videos ON comments.commentable_type = 'videos' AND comments.commentable_id = videos.id
- * // LEFT JOIN articles ON comments.commentable_type = 'articles' AND comments.commentable_id = articles.id
- * // WHERE (
- * //   (comments.commentable_type = 'posts' AND posts.title LIKE '%JavaScript%') OR
- * //   (comments.commentable_type = 'videos' AND videos.title LIKE '%JavaScript%') OR
- * //   (comments.commentable_type = 'articles' AND articles.headline LIKE '%JavaScript%')
- * // )
- *
- * @example
- * // Input: Complex polymorphic filter with cross-table paths
- * const searchSchema = {
- *   commentable_author: {
- *     type: 'string',
- *     polymorphicField: 'commentable',
- *     targetFields: {
- *       posts: 'author.name',     // Search post author's name
- *       videos: 'creator.name'    // Search video creator's name
- *     }
- *   }
- * };
- *
- * // Result: Creates nested JOINs for each polymorphic type
- * // LEFT JOIN posts ON comments.commentable_type = 'posts' AND comments.commentable_id = posts.id
- * // LEFT JOIN users AS posts_author ON posts.author_id = posts_author.id
- * // LEFT JOIN videos ON comments.commentable_type = 'videos' AND comments.commentable_id = videos.id
- * // LEFT JOIN users AS videos_creator ON videos.creator_id = videos_creator.id
- *
- * @description
- * Used by:
- * - rest-api-knex-plugin calls this hook first during query building
- * - Must run before other filter hooks to establish JOINs
- * - Applied when searchSchema contains polymorphicField definitions
- *
- * Purpose:
- * - Enables filtering on polymorphic relationships without knowing the concrete type
- * - Searches across multiple tables with a single filter parameter
- * - Supports queries like "find all comments on content containing 'JavaScript'"
- * - Uses conditional JOINs that only match when type field equals expected value
- * - Maintains performance by leveraging indexed type/id columns
- *
- * Data flow:
- * 1. Identifies filters with polymorphicField in searchSchema
- * 2. For each polymorphic type, adds conditional LEFT JOIN
- * 3. Builds OR conditions checking type field and target field together
- * 4. Sets hasJoins flag for subsequent hooks
- * 5. Returns modified query with polymorphic search capabilities
  */
 export const polymorphicFiltersHook = async (hookParams, dependencies) => {
   const { log, scopes, knex } = dependencies
@@ -222,7 +280,8 @@ export const polymorphicFiltersHook = async (hookParams, dependencies) => {
         })
 
         // Conditional JOIN - only matches when type is correct
-        query.leftJoin(`${targetTable} as ${baseAlias}`, function () {
+        const targetState = await buildSearchJoinQuery({ scopeName: targetType, scopes, context: hookParams.context, adapterUtils, db })
+        query.leftJoin(targetState.query.as(baseAlias), function () {
           const typeColumn = adapterUtils.translateColumn(scopeName, typeField, tableAlias)
           const idColumn = adapterUtils.translateColumn(scopeName, idField, tableAlias)
           const targetIdColumn = adapterUtils.translateColumn(targetType, targetIdField, baseAlias)
@@ -243,71 +302,74 @@ export const polymorphicFiltersHook = async (hookParams, dependencies) => {
           polymorphicJoins.get(baseAlias).aliasScopeMap = new Map()
         }
         polymorphicJoins.get(baseAlias).aliasScopeMap.set(baseAlias, targetType)
+      }
 
-        // Handle cross-table paths
-        if (targetFieldPath.includes('.')) {
-          log.trace('[POLYMORPHIC-SEARCH] Building cross-table JOINs for path:', targetFieldPath)
-
-          const pathParts = targetFieldPath.split('.')
-          let currentAlias = baseAlias
-          let currentScope = targetType
-
-          // Build JOIN for each segment except the last
-          for (let i = 0; i < pathParts.length - 1; i++) {
-            const relationshipName = pathParts[i]
-
-            // Find the foreign key for this relationship
-            const currentSchema = scopes[currentScope].vars.schemaInfo.schemaInstance
-            let foreignKeyField = null
-            let nextScope = null
-
-            // Search schema for matching belongsTo
-            for (const [fieldName, fieldDef] of Object.entries(currentSchema.structure)) {
-              if (fieldDef.as === relationshipName && fieldDef.belongsTo) {
-                foreignKeyField = fieldName
-                nextScope = fieldDef.belongsTo
-                break
-              }
+      // Handle cross-table paths
+      if (targetFieldPath.includes('.')) {
+        log.trace('[POLYMORPHIC-SEARCH] Building cross-table JOINs for path:', targetFieldPath)
+
+        const pathParts = targetFieldPath.split('.')
+        let currentAlias = baseAlias
+        let currentScope = targetType
+
+        // Build JOIN for each segment except the last
+        for (let i = 0; i < pathParts.length - 1; i++) {
+          const relationshipName = pathParts[i]
+
+          // Find the foreign key for this relationship
+          const currentSchema = scopes[currentScope].vars.schemaInfo.schemaInstance
+          let foreignKeyField = null
+          let nextScope = null
+
+          // Search schema for matching belongsTo
+          for (const [fieldName, fieldDef] of Object.entries(currentSchema.structure)) {
+            if (fieldDef.as === relationshipName && fieldDef.belongsTo) {
+              foreignKeyField = fieldName
+              nextScope = fieldDef.belongsTo
+              break
             }
+          }
 
-            if (!foreignKeyField) {
-              // Check relationships for hasOne
-              const currentRelationships = scopes[currentScope].vars.schemaInfo.schemaRelationships
-              const rel = currentRelationships?.[relationshipName]
-              if (rel?.hasOne) {
-                // Handle hasOne - more complex
-                throw new Error(
-                  'Cross-table polymorphic search through hasOne relationships not yet supported'
-                )
-              }
-
+          if (!foreignKeyField) {
+            // Check relationships for hasOne
+            const currentRelationships = scopes[currentScope].vars.schemaInfo.schemaRelationships
+            const rel = currentRelationships?.[relationshipName]
+            if (rel?.hasOne) {
+              // Handle hasOne - more complex
               throw new Error(
-                `Cannot resolve relationship '${relationshipName}' in path '${targetFieldPath}' for scope '${currentScope}'`
+                'Cross-table polymorphic search through hasOne relationships not yet supported'
               )
             }
 
-            // Build next JOIN
-            const nextAlias = `${currentAlias}_${relationshipName}`
-            const nextSchema = scopes[nextScope].vars.schemaInfo.schemaInstance
-            const nextTable = nextSchema?.tableName || nextScope
-
-            log.trace('[POLYMORPHIC-SEARCH] Adding cross-table JOIN:', {
-              from: currentAlias,
-              to: nextAlias,
-              table: nextTable
-            })
+            throw new Error(
+              `Cannot resolve relationship '${relationshipName}' in path '${targetFieldPath}' for scope '${currentScope}'`
+            )
+          }
 
-            const nextIdField = scopes[nextScope].vars.schemaInfo.idProperty || 'id'
-            const sourceColumn = adapterUtils.translateColumn(currentScope, foreignKeyField, currentAlias)
-            const targetColumn = adapterUtils.translateColumn(nextScope, nextIdField, nextAlias)
+          // Build next JOIN
+          const nextAlias = `${currentAlias}_${relationshipName}`
+          const nextSchema = scopes[nextScope].vars.schemaInfo.schemaInstance
+          const nextTable = nextSchema?.tableName || nextScope
 
-            query.leftJoin(`${nextTable} as ${nextAlias}`, sourceColumn, targetColumn)
+          log.trace('[POLYMORPHIC-SEARCH] Adding cross-table JOIN:', {
+            from: currentAlias,
+            to: nextAlias,
+            table: nextTable
+          })
 
-            currentAlias = nextAlias
-            currentScope = nextScope
+          const nextIdField = scopes[nextScope].vars.schemaInfo.idProperty || 'id'
+          const sourceColumn = adapterUtils.translateColumn(currentScope, foreignKeyField, currentAlias)
+          const targetColumn = adapterUtils.translateColumn(nextScope, nextIdField, nextAlias)
 
-            polymorphicJoins.get(baseAlias).aliasScopeMap.set(currentAlias, currentScope)
+          if (!polymorphicJoins.get(baseAlias).aliasScopeMap.has(nextAlias)) {
+            const nextState = await buildSearchJoinQuery({ scopeName: nextScope, scopes, context: hookParams.context, adapterUtils, db })
+            query.leftJoin(nextState.query.as(nextAlias), sourceColumn, targetColumn)
           }
+
+          currentAlias = nextAlias
+          currentScope = nextScope
+
+          polymorphicJoins.get(baseAlias).aliasScopeMap.set(currentAlias, currentScope)
         }
       }
     }
@@ -327,10 +389,24 @@ export const polymorphicFiltersHook = async (hookParams, dependencies) => {
   // Mark that we have JOINs for other hooks
   hookParams.context.knexQuery.hasJoins = true
 
+  const references = new Map()
+  for (const { fieldDef, polymorphicField } of polymorphicSearches.values()) {
+    for (const [target, path] of Object.entries(fieldDef.targetFields)) {
+      const baseAlias = `${tableAlias}_${polymorphicField}_${target}`
+      const parts = path.split('.')
+      const field = parts.pop()
+      const alias = [baseAlias, ...parts].join('_')
+      const targetScope = polymorphicJoins.get(baseAlias).aliasScopeMap.get(alias)
+      references.set(adapterUtils.translateColumn(targetScope, field, alias), { scopeName: targetScope, field, alias })
+    }
+  }
+  const columns = await resolveSearchColumns(references, { scopes, context: hookParams.context, adapterUtils, db })
+
   // Step 3: Apply WHERE conditions
   query.where(function applyPolymorphicWhere () {
     const applyComparison = (builder, scope, alias, field, operator, rawValue) => {
-      const columnRef = adapterUtils.translateColumn(scope, field, alias)
+      const qualified = adapterUtils.translateColumn(scope, field, alias)
+      const columnRef = columns.get(qualified) || qualified
       const normalizedValue = adapterUtils.translateFilterValue(scope, field, rawValue)
       applyWhereForOperator({ builder, columnRef, operator, value: normalizedValue, knex })
     }
@@ -351,6 +427,7 @@ export const polymorphicFiltersHook = async (hookParams, dependencies) => {
             applyComparison(this, scopeName, typeColumnAlias, typeField, '=', targetType)
 
             const baseAlias = `${tableAlias}_${searchInfo.polymorphicField}_${targetType}`
+            this.whereNotNull(adapterUtils.translateColumn(targetType, 'id', baseAlias))
             const joinMeta = polymorphicJoins.get(baseAlias)
             const aliasScopeMap = joinMeta?.aliasScopeMap || new Map([[baseAlias, targetType]])
 
@@ -383,92 +460,6 @@ export const polymorphicFiltersHook = async (hookParams, dependencies) => {
  *
  * @param {Object} hookParams - Hook parameters containing context
  * @param {Object} dependencies - Dependencies injected by the plugin
- *
- * @example
- * // Input: Simple cross-table filter
- * const searchSchema = {
- *   author_name: {
- *     type: 'string',
- *     actualField: 'author.name',  // Dot notation indicates JOIN needed
- *     filterOperator: 'like'
- *   }
- * };
- *
- * // Filter request: { author_name: 'Smith' }
- * // Query before: SELECT * FROM articles
- *
- * // Result: Adds JOIN and qualified WHERE
- * // Query after:
- * // SELECT * FROM articles
- * // LEFT JOIN users AS articles_author ON articles.author_id = articles_author.id
- * // WHERE articles_author.name LIKE '%Smith%'
- *
- * @example
- * // Input: Multi-field search across tables
- * const searchSchema = {
- *   search: {
- *     type: 'string',
- *     oneOf: [
- *       'title',           // Local field
- *       'content',         // Local field
- *       'author.name',     // Requires JOIN to users
- *       'category.title'   // Requires JOIN to categories
- *     ],
- *     filterOperator: 'like'
- *   }
- * };
- *
- * // Filter request: { search: 'JavaScript' }
- *
- * // Result: Multiple JOINs and OR conditions
- * // SELECT DISTINCT * FROM articles
- * // LEFT JOIN users AS articles_author ON articles.author_id = articles_author.id
- * // LEFT JOIN categories AS articles_category ON articles.category_id = articles_category.id
- * // WHERE (
- * //   articles.title LIKE '%JavaScript%' OR
- * //   articles.content LIKE '%JavaScript%' OR
- * //   articles_author.name LIKE '%JavaScript%' OR
- * //   articles_category.title LIKE '%JavaScript%'
- * // )
- *
- * @example
- * // Input: Deep nested relationships (3 levels)
- * const searchSchema = {
- *   company_country: {
- *     type: 'string',
- *     actualField: 'author.company.country.name'
- *   }
- * };
- *
- * // Filter request: { company_country: 'USA' }
- *
- * // Result: Chain of JOINs following relationships
- * // SELECT * FROM articles
- * // LEFT JOIN users AS articles_author ON articles.author_id = articles_author.id
- * // LEFT JOIN companies AS articles_author_company ON articles_author.company_id = articles_author_company.id
- * // LEFT JOIN countries AS articles_author_company_country ON articles_author_company.country_id = articles_author_company_country.id
- * // WHERE articles_author_company_country.name = 'USA'
- *
- * @description
- * Used by:
- * - rest-api-knex-plugin calls this hook second during query building
- * - Runs after polymorphicFiltersHook but before basicFiltersHook
- * - Applied when filters contain dot notation in actualField or oneOf
- *
- * Purpose:
- * - Enables filtering on related table fields without manual JOIN writing
- * - Automatically builds JOIN chains from dot notation paths
- * - Detects one-to-many relationships and adds DISTINCT to prevent duplicates
- * - Creates unique aliases to avoid naming conflicts
- * - Validates that target fields are indexed for performance
- *
- * Data flow:
- * 1. Scans filters for dot notation in actualField or oneOf arrays
- * 2. For each cross-table reference, builds JOIN chain via buildJoinChain
- * 3. Applies JOINs to query with proper aliasing
- * 4. Adds DISTINCT if any one-to-many JOINs detected
- * 5. Applies WHERE conditions using qualified field names
- * 6. Sets hasJoins flag for basicFiltersHook to use
  */
 export const crossTableFiltersHook = async (hookParams, dependencies) => {
   const { log, scopes, knex } = dependencies
@@ -489,13 +480,7 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
   const aliasScopeMap = new Map()
   aliasScopeMap.set(tableAlias, scopeName)
 
-  // Step 1: Analyze indexes
-  const requiredIndexes = analyzeRequiredIndexes(scopes, log, scopeName, schemaInfo)
-  if (requiredIndexes.length > 0) {
-    log.debug('Cross-table search requires indexes:', requiredIndexes)
-  }
-
-  // Step 2: Build JOIN maps
+  // Build JOIN maps.
   const joinMap = new Map()
   const fieldPathMap = new Map()
   let hasCrossTableFilters = false
@@ -562,13 +547,7 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
     return adapterUtils.translateColumn(scopeForAlias, field, alias)
   }
 
-  const resolveFieldColumn = (field) => {
-    if (field.includes('.')) {
-      const qualified = fieldPathMap.get(field) || field
-      return translateQualifiedColumn(qualified)
-    }
-    return adapterUtils.translateColumn(scopeName, field, tableAlias)
-  }
+  const resolveFieldColumn = field => columns.get(field)
 
   const normalizeFieldValue = (field, value) => {
     if (field.includes('.')) {
@@ -587,11 +566,11 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
     return adapterUtils.translateFilterValue(scopeName, field, value)
   }
 
-  // Step 3: Apply JOINs
+  // Apply JOINs.
   const appliedJoins = new Set()
 
-  const applyPolymorphicJoin = (join) => {
-    query.leftJoin(`${join.targetTableName} as ${join.joinAlias}`, function () {
+  const applyPolymorphicJoin = (join, source) => {
+    query.leftJoin(source.as(join.joinAlias), function () {
       const parts = join.joinCondition.split(' AND ')
       const [typeCondition, idCondition] = parts
 
@@ -617,37 +596,40 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
     })
   }
 
-  const applyStandardJoin = (join) => {
+  const applyStandardJoin = (join, source) => {
     const [leftSide, rightSide] = join.joinCondition.split(' = ')
     const translatedLeft = translateQualifiedColumn(leftSide.trim())
     const translatedRight = translateQualifiedColumn(rightSide.trim())
-    query.leftJoin(`${join.targetTableName} as ${join.joinAlias}`, function () {
+    query.leftJoin(source.as(join.joinAlias), function () {
       this.on(translatedLeft, translatedRight)
     })
   }
 
-  const processJoin = (join) => {
+  const processJoin = async (join) => {
     const joinKey = `${join.joinAlias}:${join.joinCondition}`
     if (appliedJoins.has(joinKey)) return
 
+    const state = join.relationshipType === 'manyToMany_pivot' && dependencies.buildSearchMembershipQuery
+      ? await dependencies.buildSearchMembershipQuery({ ...join, db, context: hookParams.context })
+      : await buildSearchJoinQuery({ scopeName: join.targetScopeName, scopes, context: hookParams.context, adapterUtils, db })
     if (join.isPolymorphic && join.joinCondition.includes(' AND ')) {
-      applyPolymorphicJoin(join)
+      applyPolymorphicJoin(join, state.query)
     } else {
-      applyStandardJoin(join)
+      applyStandardJoin(join, state.query)
     }
 
     appliedJoins.add(joinKey)
   }
 
-  joinMap.forEach((joinInfo) => {
+  for (const joinInfo of joinMap.values()) {
     if (joinInfo.isMultiLevel && Array.isArray(joinInfo.joinChain)) {
-      joinInfo.joinChain.forEach(processJoin)
+      for (const join of joinInfo.joinChain) await processJoin(join)
     } else {
-      processJoin(joinInfo)
+      await processJoin(joinInfo)
     }
-  })
+  }
 
-  // Step 4: Handle DISTINCT
+  // Handle DISTINCT.
   let hasOneToManyJoins = false
   joinMap.forEach((joinInfo) => {
     if (joinInfo.isOneToMany) {
@@ -667,6 +649,22 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
   // Store state for basic filters hook
   hookParams.context.knexQuery.hasJoins = true
 
+  const references = new Map()
+  for (const filterKey of Object.keys(filters)) {
+    const fieldDef = schemaInfo.searchSchemaStructure[filterKey]
+    if (!fieldDef || fieldDef.polymorphicField ||
+      (!fieldDef.actualField?.includes('.') && !fieldDef.oneOf?.some(field => field.includes('.')))) continue
+    for (const field of fieldDef.oneOf || [fieldDef.actualField || filterKey]) {
+      if (field.includes('.')) {
+        const [alias, ...parts] = (fieldPathMap.get(field) || field).split('.')
+        references.set(field, { scopeName: aliasScopeMap.get(alias) || scopeName, field: parts.join('.'), alias })
+      } else {
+        references.set(field, { scopeName, field, alias: tableAlias })
+      }
+    }
+  }
+  const columns = await resolveSearchColumns(references, { scopes, context: hookParams.context, adapterUtils, db })
+
   // Step 5: Apply WHERE conditions for cross-table filters
   query.where(function () {
     for (const [filterKey, filterValue] of Object.entries(filters)) {
@@ -688,8 +686,6 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
           let searchTerms = [filterValue]
           if (fieldDef.splitBy && typeof filterValue === 'string') {
             searchTerms = filterValue.split(fieldDef.splitBy).filter(term => term.trim())
-          } else if (Array.isArray(filterValue)) {
-            searchTerms = filterValue
           }
 
           const applyTermComparison = (builder, method, field, raw) => {
@@ -706,27 +702,25 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
           }
 
           this.where(function () {
-            if (fieldDef.matchAll && searchTerms.length > 1) {
-              searchTerms.forEach(term => {
-                this.andWhere(function () {
-                  fieldDef.oneOf.forEach((field, index) => {
-                    const method = index === 0 ? 'and' : 'or'
-                    applyTermComparison(this, method, field, term)
-                  })
+            searchTerms.forEach((term, termIndex) => {
+              const join = fieldDef.matchAll || termIndex === 0 ? 'where' : 'orWhere'
+              this[join](function () {
+                fieldDef.oneOf.forEach((field, index) => {
+                  applyTermComparison(this, index === 0 ? 'and' : 'or', field, term)
                 })
               })
-            } else {
-              fieldDef.oneOf.forEach((field, index) => {
-                const method = index === 0 ? 'and' : 'or'
-                applyTermComparison(this, method, field, filterValue)
-              })
-            }
+            })
           })
           break
         }
 
         case fieldDef.applyFilter && typeof fieldDef.applyFilter === 'function':
-          fieldDef.applyFilter.call(this, this, filterValue)
+          fieldDef.applyFilter.call(this, this, filterValue, {
+            column: field => translateQualifiedColumn(fieldPathMap.get(field) || field),
+            value: normalizeFieldValue,
+            context: hookParams.context,
+            scopeName
+          })
           break
 
         default: {
@@ -747,141 +741,6 @@ export const crossTableFiltersHook = async (hookParams, dependencies) => {
  *
  * @param {Object} hookParams - Hook parameters containing context
  * @param {Object} dependencies - Dependencies injected by the plugin
- *
- * @example
- * // Input: Basic equality filter
- * const searchSchema = {
- *   status: {
- *     type: 'string',
- *     filterOperator: '='  // Default is '=' if not specified
- *   }
- * };
- *
- * // Filter request: { status: 'published' }
- * // Query before: SELECT * FROM articles
- *
- * // Result: Adds qualified WHERE clause
- * // Query after: SELECT * FROM articles WHERE articles.status = 'published'
- *
- * @example
- * // Input: LIKE filter for partial text matching
- * const searchSchema = {
- *   title: {
- *     type: 'string',
- *     filterOperator: 'like'
- *   },
- *   content: {
- *     type: 'string',
- *     filterOperator: 'like'
- *   }
- * };
- *
- * // Filter request: { title: 'JavaScript', content: 'async' }
- *
- * // Result: Multiple LIKE conditions
- * // WHERE articles.title LIKE '%JavaScript%'
- * // AND articles.content LIKE '%async%'
- *
- * @example
- * // Input: Multi-field OR search with oneOf
- * const searchSchema = {
- *   search: {
- *     type: 'string',
- *     oneOf: ['title', 'content', 'summary'],
- *     filterOperator: 'like',
- *     splitBy: ' ',        // Split search terms by space
- *     matchAll: true       // All terms must match somewhere
- *   }
- * };
- *
- * // Filter request: { search: 'REST API' }
- *
- * // Result: Each term must match in at least one field
- * // WHERE (
- * //   (articles.title LIKE '%REST%' OR articles.content LIKE '%REST%' OR articles.summary LIKE '%REST%')
- * //   AND
- * //   (articles.title LIKE '%API%' OR articles.content LIKE '%API%' OR articles.summary LIKE '%API%')
- * // )
- *
- * @example
- * // Input: Advanced operators - IN and BETWEEN
- * const searchSchema = {
- *   category_id: {
- *     type: 'array',
- *     filterOperator: 'in'
- *   },
- *   price: {
- *     type: 'array',
- *     filterOperator: 'between'
- *   },
- *   tags: {
- *     type: 'array',
- *     filterOperator: 'in'
- *   }
- * };
- *
- * // Filter request: {
- * //   category_id: [1, 2, 3],
- * //   price: [10.00, 99.99],
- * //   tags: ['javascript', 'nodejs']
- * // }
- *
- * // Result: IN and BETWEEN clauses
- * // WHERE articles.category_id IN (1, 2, 3)
- * // AND articles.price BETWEEN 10.00 AND 99.99
- * // AND articles.tags IN ('javascript', 'nodejs')
- *
- * @example
- * // Input: Custom filter function
- * const searchSchema = {
- *   has_comments: {
- *     type: 'boolean',
- *     applyFilter: function(query, value) {
- *       if (value === true) {
- *         query.whereExists(function() {
- *           this.select('id')
- *               .from('comments')
- *               .whereRaw('comments.article_id = articles.id');
- *         });
- *       } else if (value === false) {
- *         query.whereNotExists(function() {
- *           this.select('id')
- *               .from('comments')
- *               .whereRaw('comments.article_id = articles.id');
- *         });
- *       }
- *     }
- *   }
- * };
- *
- * // Filter request: { has_comments: true }
- *
- * // Result: Subquery to check existence
- * // WHERE EXISTS (
- * //   SELECT id FROM comments WHERE comments.article_id = articles.id
- * // )
- *
- * @description
- * Used by:
- * - rest-api-knex-plugin calls this hook last during query building
- * - Runs after all JOINs have been established by previous hooks
- * - Handles all non-cross-table, non-polymorphic filters
- *
- * Purpose:
- * - Implements standard SQL filtering operations safely via Knex
- * - Always qualifies field names with table name to prevent ambiguity
- * - Supports various operators: =, like, in, between, and custom
- * - Enables multi-field OR searches with optional term splitting
- * - Handles null values appropriately (using whereNull)
- * - Allows custom filter logic via applyFilter functions
- *
- * Data flow:
- * 1. Skips filters already handled by polymorphic/cross-table hooks
- * 2. Qualifies all field names with table name (e.g., articles.title)
- * 3. Applies appropriate WHERE clause based on filterOperator
- * 4. For oneOf filters, creates OR conditions across specified fields
- * 5. For custom filters, delegates to applyFilter function
- * 6. Returns query with all basic filters applied
  */
 export const basicFiltersHook = async (hookParams, dependencies) => {
   const { log, scopes, knex } = dependencies
@@ -891,6 +750,7 @@ export const basicFiltersHook = async (hookParams, dependencies) => {
   const scopeName = hookParams.context?.knexQuery?.scopeName
   const filters = hookParams.context?.knexQuery?.filters
   const query = hookParams.context?.knexQuery?.query
+  const db = hookParams.context?.knexQuery?.db || knex
   if (!scopeName || !filters || !scopes[scopeName]) {
     return
   }
@@ -902,14 +762,18 @@ export const basicFiltersHook = async (hookParams, dependencies) => {
   log.trace('[DEBUG basicFiltersHook] Called with:', {
     scopeName,
     hasFilters: !!filters,
-    filters,
-    searchSchemaKeys: Object.keys(schemaInfo.searchSchemaStructure || {}),
     tableName
   })
 
-  // Check if we have any JOINs applied (to know if we need to qualify fields)
-  // Instead of relying on hasJoins flag, always qualify fields for safety
-  const qualifyField = (field) => adapterUtils.translateColumn(scopeName, field, tableAlias)
+  const references = new Map()
+  for (const filterKey of Object.keys(filters)) {
+    const fieldDef = schemaInfo.searchSchemaStructure[filterKey]
+    if (!fieldDef || fieldDef.polymorphicField || fieldDef.actualField?.includes('.') ||
+      fieldDef.oneOf?.some(field => field.includes('.')) || (fieldDef.applyFilter && !fieldDef.oneOf)) continue
+    for (const field of fieldDef.oneOf || [fieldDef.actualField || filterKey]) references.set(field, { scopeName, field, alias: tableAlias })
+  }
+  const columns = await resolveSearchColumns(references, { scopes, context: hookParams.context, adapterUtils, db })
+  const qualifyField = field => columns.get(field)
   const normalizeValue = (field, value) => {
     if (Array.isArray(value)) {
       return value.map((entry) => adapterUtils.translateFilterValue(scopeName, field, entry))
@@ -925,7 +789,7 @@ export const basicFiltersHook = async (hookParams, dependencies) => {
         log.trace(`[DEBUG basicFiltersHook] No field definition for filter key: ${filterKey}`)
         continue
       }
-      log.trace(`[DEBUG basicFiltersHook] Processing filter: ${filterKey} = ${filterValue}, fieldDef:`, fieldDef)
+      log.trace(`[DEBUG basicFiltersHook] Processing filter: ${filterKey}`)
 
       // Skip if this is a cross-table filter
       if (fieldDef.actualField?.includes('.') ||
@@ -948,47 +812,33 @@ export const basicFiltersHook = async (hookParams, dependencies) => {
           }
 
           this.where(function () {
-            if (fieldDef.matchAll && searchTerms.length > 1) {
-              searchTerms.forEach(term => {
-                const normalizedTerm = normalizeValue(fieldDef.oneOf[0], term)
-                this.andWhere(function () {
-                  fieldDef.oneOf.forEach((field, index) => {
-                    const columnRef = qualifyField(field)
-                    const perFieldValue = Array.isArray(normalizedTerm)
-                      ? normalizeValue(field, term)
-                      : normalizeValue(field, term)
-                    applyWhereForOperator({
-                      builder: this,
-                      columnRef,
-                      operator,
-                      value: perFieldValue,
-                      knex,
-                      or: index !== 0,
-                    })
+            searchTerms.forEach((term, termIndex) => {
+              const join = fieldDef.matchAll || termIndex === 0 ? 'where' : 'orWhere'
+              this[join](function () {
+                fieldDef.oneOf.forEach((field, index) => {
+                  applyWhereForOperator({
+                    builder: this,
+                    columnRef: qualifyField(field),
+                    operator,
+                    value: normalizeValue(field, term),
+                    knex,
+                    or: index !== 0,
                   })
                 })
               })
-            } else {
-              fieldDef.oneOf.forEach((field, index) => {
-                const columnRef = qualifyField(field)
-                const normalizedValue = normalizeValue(field, filterValue)
-                applyWhereForOperator({
-                  builder: this,
-                  columnRef,
-                  operator,
-                  value: normalizedValue,
-                  knex,
-                  or: index !== 0,
-                })
-              })
-            }
+            })
           })
           break
         }
 
         case fieldDef.applyFilter && typeof fieldDef.applyFilter === 'function':
           // Custom filter
-          fieldDef.applyFilter.call(this, this, filterValue)
+          fieldDef.applyFilter.call(this, this, filterValue, {
+            column: field => adapterUtils.translateColumn(scopeName, field, tableAlias),
+            value: normalizeValue,
+            context: hookParams.context,
+            scopeName
+          })
           break
 
         default: {
@@ -999,7 +849,7 @@ export const basicFiltersHook = async (hookParams, dependencies) => {
 
           const operator = resolveSearchOperator(fieldDef)
           const normalized = normalizeValue(actualField, filterValue)
-          log.trace(`[DEBUG basicFiltersHook] Applying filter: ${dbField} ${operator} ${filterValue}`)
+          log.trace(`[DEBUG basicFiltersHook] Applying filter: ${dbField} ${operator}`)
           applyWhereForOperator({ builder: this, columnRef: dbField, operator, value: normalized, knex })
           break
         }