praxis 2.0.pre.8 → 2.0.pre.13

data/lib/praxis/extensions/attribute_filtering/active_record_filter_query_builder.rb

@@ -5,58 +5,166 @@ module Praxis
     module AttributeFiltering
       ALIAS_TABLE_PREFIX = ''
       require_relative 'active_record_patches'
+      # Helper class that can present an SqlLiteral string which we have already quoted
+      # ... but! that can properly provide a "to_sym" that has the value unquoted
+      # This is necessary as (the latest AR code):
+      # * does not carry over "references" in joins if they are not SqlLiterals
+      # * but, at the same time, it indexes the references using the .to_sym value (which is really expected to be the normal string, without quotes)
+      # If we pass a normal SqlLiteral, instead of our wrapper, without quoting the table, the current AR code will never quote it to form the 
+      # SQL string, as it's already a literal...so our "/" type separators as names won't work without quoting.
+      class QuasiSqlLiteral < Arel::Nodes::SqlLiteral
+        def initialize(quoted:, symbolized:)
+          @symbolized = symbolized
+          super(quoted)
+        end
+        def to_sym
+          @symbolized
+        end
+      end
 
       class ActiveRecordFilterQueryBuilder
-      attr_reader :query, :model, :attr_to_column
+      attr_reader :model, :filters_map
 
         # Base query to build upon
         def initialize(query: , model:, filters_map:, debug: false)
-          @query = query
+          # Note: Do not make the initial_query an attr reader to make sure we don't count/leak on modifying it. Easier to mostly use class methods
+          @initial_query = query
           @model = model
-          @attr_to_column = filters_map
+          @filters_map = filters_map
           @logger = debug ? Logger.new(STDOUT) : nil
+          @active_record_version_maj = ActiveRecord.gem_version.segments[0]
         end
         
-        def debug(msg)
-          @logger && @logger.puts(msg)
+        def debug_query(msg, query)
+          @logger.info(msg + query.to_sql) if @logger
         end
 
         def generate(filters)
           # Resolve the names and values first, based on filters_map
           root_node = _convert_to_treenode(filters)
-          craft_filter_query(root_node, for_model: @model)
-          debug("SQL due to filters: #{@query.all.to_sql}")
-          @query
+          crafted = craft_filter_query(root_node, for_model: @model)
+          debug_query("SQL due to filters: ", crafted.all)
+          crafted
         end
 
         def craft_filter_query(nodetree, for_model:)
           result = _compute_joins_and_conditions_data(nodetree, model: for_model)
-          @query = query.joins(result[:associations_hash]) unless result[:associations_hash].empty?
+          return @initial_query if result[:conditions].empty?
+
+          
+          # Find the root group (usually an AND group) but can be an OR group, or nil if there's only 1 condition
+          root_parent_group = result[:conditions].first[:node_object].parent_group || result[:conditions].first[:node_object]
+          while root_parent_group.parent_group != nil
+            root_parent_group = root_parent_group.parent_group
+          end
 
-          result[:conditions].each do |condition|
-            filter_name = condition[:name]
-            filter_value = condition[:value]
+          # Process the joins
+          query_with_joins = result[:associations_hash].empty? ? @initial_query : @initial_query.joins(result[:associations_hash])
+
+          # Proc to apply a single condition
+          apply_single_condition = Proc.new do |condition, associated_query|
+            colo = condition[:model].columns_hash[condition[:name].to_s]
             column_prefix = condition[:column_prefix]
+            #Mark where clause referencing the appropriate alias
+            associated_query = associated_query.references(build_reference_value(column_prefix, query: associated_query))
+            self.class.add_clause(
+              query: associated_query, 
+              column_prefix: column_prefix, 
+              column_object: colo, 
+              op: condition[:op], 
+              value: condition[:value]
+            )
+          end
 
-            colo = condition[:model].columns_hash[filter_name.to_s]
-            add_clause(column_prefix: column_prefix,  column_object: colo, op: condition[:op], value: filter_value)
+          if @active_record_version_maj < 6
+            # ActiveRecord < 6 does not support '.and' so no nested things can be done
+            # But we can still support the case of 1+ flat conditions of the same AND/OR type
+            if root_parent_group.is_a?(FilteringParams::Condition)
+              # A Single condition it is easy to handle
+              apply_single_condition.call(result[:conditions].first, query_with_joins)
+            elsif root_parent_group.items.all?{|i| i.is_a?(FilteringParams::Condition)}
+              # Only 1 top level root, with only with simple condition items
+              if root_parent_group.type == :and
+                result[:conditions].reverse.inject(query_with_joins) do |accum, condition|
+                  apply_single_condition.call(condition, accum)
+                end
+              else
+                # To do a flat OR, we need to apply the first condition to the incoming query
+                # and then apply any extra ORs to it. Otherwise Book.or(X).or(X) still matches all books
+                cond1, *rest = result[:conditions].reverse
+                start_query = apply_single_condition.call(cond1, query_with_joins)
+                rest.inject(start_query) do |accum, condition|
+                  accum.or(apply_single_condition.call(condition, query_with_joins))
+                end
+              end
+            else
+              raise "Mixing AND and OR conditions is not supported for ActiveRecord <6."
+            end
+          else #  ActiveRecord 6+
+            # Process the conditions in a depth-first order, and return the resulting query
+            _depth_first_traversal(
+              root_query: query_with_joins, 
+              root_node: root_parent_group, 
+              conditions: result[:conditions], 
+              &apply_single_condition
+            )
           end
         end
 
         private
+        def _depth_first_traversal(root_query:, root_node:, conditions:, &block)
+          # Save the associated query for non-leaves 
+          root_node.associated_query = root_query if root_node.is_a?(FilteringParams::ConditionGroup)
+          
+          if root_node.is_a?(FilteringParams::Condition)
+            matching_condition = conditions.find {|cond| cond[:node_object] == root_node }
+
+            # The simplified case of a single top level condition (without a wrapping group)
+            # will need to pass the root query itself
+            associated_query = root_node.parent_group ? root_node.parent_group.associated_query : root_query
+            return yield matching_condition, associated_query
+          else
+            first_query, *rest_queries = root_node.items.map do |child|
+              _depth_first_traversal(root_query: root_query, root_node: child, conditions: conditions, &block)
+            end
+
+            rest_queries.each.inject(first_query) do |q, a_query|
+              root_node.type == :and ? q.and(a_query) : q.or(a_query)
+            end
+          end
+        end
+
+        def _mapped_filter(name)
+          target = @filters_map[name]
+          unless target
+            if @model.attribute_names.include?(name.to_s)
+              # Cache it in the filters mapping (to avoid later lookups), and return it.
+              @filters_map[name] = name
+              target = name
+            end
+          end
+          return target
+        end
 
         # Resolve and convert from filters, to a more manageable and param-type-independent structure
         def _convert_to_treenode(filters)
           # Resolve the names and values first, based on filters_map
           resolved_array = []
           filters.parsed_array.each do |filter|
-            mapped_value = attr_to_column[filter[:name]]
-            raise "Filtering by #{filter[:name]} not allowed (no mapping found)" unless mapped_value
+            mapped_value = _mapped_filter(filter[:name])
+            unless mapped_value
+              msg = "Filtering by #{filter[:name]} is not allowed. No implementation mapping defined for it has been found \
+                and there is not a model attribute with this name either.\n" \
+                "Please add a mapping for #{filter[:name]} in the `filters_mapping` method of the appropriate Resource class"
+              raise msg
+            end
             bindings_array = \
               if mapped_value.is_a?(Proc)
                 result = mapped_value.call(filter)
                 # Result could be an array of hashes (each hash has name/op/value to identify a condition)
-                result.is_a?(Array) ? result : [result]
+                result_from_proc = result.is_a?(Array) ? result : [result]
+                # Make sure we tack on the node object associated with the filter
+                result_from_proc.map{|hash| hash.merge(node_object: filter[:node_object])}
               else
                 # For non-procs there's only 1 filter and 1 value (we're just overriding the mapped value)
                 [filter.merge( name: mapped_value)]
@@ -84,51 +192,51 @@ module Praxis
           {associations_hash: h, conditions: conditions}
         end
 
-        def add_clause(column_prefix:, column_object:, op:, value:)
-          @query = @query.references(column_prefix) #Mark where clause referencing the appropriate alias
+        def self.add_clause(query:, column_prefix:, column_object:, op:, value:)
           likeval = get_like_value(value)
           case op
-            when '!' # name! means => name IS NOT NULL (and the incoming value is nil)
-              op = '!='
-              value = nil # Enforce it is indeed nil (should be)
-            when '!!'
-              op = '='
-              value = nil # Enforce it is indeed nil (should be)
+          when '!' # name! means => name IS NOT NULL (and the incoming value is nil)
+            op = '!='
+            value = nil # Enforce it is indeed nil (should be)
+          when '!!'
+            op = '='
+            value = nil # Enforce it is indeed nil (should be)
+          end
+
+          case op
+          when '='
+            if likeval
+              add_safe_where(query: query, tab: column_prefix, col: column_object, op: 'LIKE', value: likeval)
+            else
+              quoted_right = quote_right_part(query: query, value: value, column_object: column_object, negative: false)
+              query.where("#{quote_column_path(query: query, prefix: column_prefix, column_object: column_object)} #{quoted_right}")
+            end
+          when '!='
+            if likeval
+              add_safe_where(query: query, tab: column_prefix, col: column_object, op: 'NOT LIKE', value: likeval)
+            else
+              quoted_right = quote_right_part(query: query, value: value, column_object: column_object, negative: true)
+              query.where("#{quote_column_path(query: query, prefix: column_prefix, column_object: column_object)} #{quoted_right}")
             end
-          @query =  case op
-                    when '='
-                      if likeval
-                        add_safe_where(tab: column_prefix, col: column_object, op: 'LIKE', value: likeval)
-                      else
-                        quoted_right = quote_right_part(value: value, column_object: column_object, negative: false)
-                        query.where("#{quote_column_path(column_prefix, column_object)} #{quoted_right}")
-                      end
-                    when '!='
-                      if likeval
-                        add_safe_where(tab: column_prefix, col: column_object, op: 'NOT LIKE', value: likeval)
-                      else
-                        quoted_right = quote_right_part(value: value, column_object: column_object, negative: true)
-                        query.where("#{quote_column_path(column_prefix, column_object)} #{quoted_right}")
-                      end
-                    when '>'
-                      add_safe_where(tab: column_prefix, col: column_object, op: '>', value: value)
-                    when '<'
-                      add_safe_where(tab: column_prefix, col: column_object, op: '<', value: value)
-                    when '>='
-                      add_safe_where(tab: column_prefix, col: column_object, op: '>=', value: value)
-                    when '<='
-                      add_safe_where(tab: column_prefix, col: column_object, op: '<=', value: value)
-                    else
-                      raise "Unsupported Operator!!! #{op}"
-                    end
-        end
-
-        def add_safe_where(tab:, col:, op:, value:)
+          when '>'
+            add_safe_where(query: query, tab: column_prefix, col: column_object, op: '>', value: value)
+          when '<'
+            add_safe_where(query: query, tab: column_prefix, col: column_object, op: '<', value: value)
+          when '>='
+            add_safe_where(query: query, tab: column_prefix, col: column_object, op: '>=', value: value)
+          when '<='
+            add_safe_where(query: query, tab: column_prefix, col: column_object, op: '<=', value: value)
+          else
+            raise "Unsupported Operator!!! #{op}"
+          end
+        end
+
+        def self.add_safe_where(query:, tab:, col:, op:, value:)
           quoted_value = query.connection.quote_default_expression(value,col)
-          query.where("#{quote_column_path(tab, col)} #{op} #{quoted_value}")
+          query.where("#{self.quote_column_path(query: query, prefix: tab, column_object: col)} #{op} #{quoted_value}")
         end
 
-        def quote_column_path(prefix, column_object)
+        def self.quote_column_path(query:, prefix:, column_object:)
           c = query.connection
           quoted_column = c.quote_column_name(column_object.name)
           if prefix
@@ -139,7 +247,7 @@ module Praxis
           end
         end
 
-        def quote_right_part(value:, column_object:, negative:)
+        def self.quote_right_part(query:, value:, column_object:, negative:)
           conn = query.connection
           if value.nil?
             no = negative ? ' NOT' : ''
@@ -157,7 +265,7 @@ module Praxis
         end
 
         # Returns nil if the value was not a fuzzzy pattern
-        def get_like_value(value)
+        def self.get_like_value(value)
           if value.is_a?(String) && (value[-1] == '*' || value[0] == '*')
             likeval = value.dup
             likeval[-1] = '%' if value[-1] == '*'
@@ -165,6 +273,22 @@ module Praxis
             likeval
           end
         end
+
+        # The value that we need to stick in the references method is different in the latest Rails
+        maj, min, _ = ActiveRecord.gem_version.segments
+        if maj == 5 || (maj == 6 && min == 0)
+          # In AR 6 (and 6.0) the references are simple strings
+          def build_reference_value(column_prefix, query: nil)
+            column_prefix
+          end
+        else
+          # The latest AR versions discard passing references to joins when they're not SqlLiterals ... so let's wrap it
+          # with our class, so that it is a literal (already quoted), but that can still provide the expected "symbol" without quotes
+          # so that our aliasing code can match it.
+          def build_reference_value(column_prefix, query:)
+            QuasiSqlLiteral.new(quoted: query.connection.quote_table_name(column_prefix), symbolized: column_prefix.to_sym)
+          end
+        end
       end
     end
   end

data/lib/praxis/extensions/attribute_filtering/filter_tree_node.rb

@@ -3,7 +3,8 @@ module Praxis
     module AttributeFiltering
       class FilterTreeNode
         attr_reader :path, :conditions, :children
-        #   # parsed_filters is an Array of {name: X, op: , value: } ... exactly the format of the FilteringParams.load method
+        # Parsed_filters is an Array of {name: X, op: Y, value: Z} ... exactly the format of the FilteringParams.load method
+        # It can also contain a :node_object
         def initialize(parsed_filters, path: [])
           @path = path # Array that marks the tree 'path' to this node (with respect to the absolute root)
           @conditions = [] # Conditions to apply directly to this node
@@ -14,7 +15,7 @@ module Praxis
             if components.empty?
               return
             elsif components.size == 1
-              @conditions << hash.slice(:name, :op, :value)
+              @conditions << hash.slice(:name, :op, :value, :node_object)
             else
               children_data[components.first] ||= []
               children_data[components.first] << hash

data/lib/praxis/extensions/attribute_filtering/filtering_params.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
-# rubocop:disable all
+require 'praxis/extensions/attribute_filtering/filters_parser'
+
 #
-# Attributor type to define and handlea simple language to express filtering attributes in listings.
+# Attributor type to define and handle the language to express filtering attributes in listings.
 # Commonly used in a query string parameter value for listing calls.
 # 
 # The type allows you to restrict the allowable fields (and their types) based on an existing Mediatype.
@@ -14,7 +15,7 @@
 # attribute :filters,
 #           Types::FilteringParams.for(MediaTypes::MyType) do
 #   filter 'user.id', using: ['=', '!=']
-#   filter 'name', using: ['=', '!=']
+#   filter 'name', using: ['=', '!=', '!', '!!]
 #   filter 'children.created_at', using: ['>', '>=', '<', '<=']
 #   filter 'display_name', using: ['=', '!='], fuzzy: true
 # end
@@ -26,6 +27,8 @@ module Praxis
         include Attributor::Type
         include Attributor::Dumpable
   
+        attr_reader :parsed_array
+
         class DSLCompiler < Attributor::DSLCompiler
           # "account.id": { operators: ["=", "!="] },
           # name:         { operators: ["=", "!="], fuzzy_match: true },
@@ -36,9 +39,9 @@ module Praxis
           end
         end
   
-        VALUE_REGEX = /[^,&]*/
-        AVAILABLE_OPERATORS = Set.new(['!=', '>=', '<=', '=', '<', '>','!','!!']).freeze
-        FILTER_REGEX = /(?<attribute>([^=!><])+)(?<operator>!=|>=|<=|!!|=|<|>|!)(?<value>#{VALUE_REGEX}(,#{VALUE_REGEX})*)/
+        VALUE_OPERATORS = Set.new(['!=', '>=', '<=', '=', '<', '>']).freeze
+        NOVALUE_OPERATORS = Set.new(['!','!!']).freeze
+        AVAILABLE_OPERATORS = Set.new(VALUE_OPERATORS+NOVALUE_OPERATORS).freeze
   
         # Abstract class, which needs to be used by subclassing it through the .for method, to set the allowed filters
         # definition should be a hash, keyed by field name, which contains a hash that can have two pieces of metadata
@@ -52,7 +55,7 @@ module Praxis
           def for(media_type, **_opts)
             unless media_type < Praxis::MediaType
               raise ArgumentError, "Invalid type: #{media_type.name} for Filters. " \
-                'Must be a subclass of MediaType'
+                'Using the .for method for defining a filter, requires passing a subclass of a MediaType'
             end
   
             ::Class.new(self) do
@@ -77,9 +80,7 @@ module Praxis
             }
           end
         end
-  
-        attr_reader :parsed_array
-  
+    
         def self.native_type
           self
         end
@@ -103,7 +104,7 @@ module Praxis
         def self.construct(definition, **options)
           return self if definition.nil?
   
-          DSLCompiler.new(self, options).parse(*definition)
+          DSLCompiler.new(self, **options).parse(*definition)
           self
         end
   
@@ -134,11 +135,15 @@ module Praxis
                          raise "filter with name #{filter_name} does not correspond to an existing field inside " \
                                " MediaType #{media_type.name}"
                        end
-                       attr_example = filter_components.inject(mt_example) do |last, name|
-                         # we can safely do sends, since we've verified the components are valid
-                         last.send(name)
-                       end
-                       arr << "#{filter_name}#{op}#{attr_example}"
+                       if NOVALUE_OPERATORS.include?(op)
+                        arr << "#{filter_name}#{op}" # Do not add a value for the operators that don't take it
+                       else
+                        attr_example = filter_components.inject(mt_example) do |last, name|
+                          # we can safely do sends, since we've verified the components are valid
+                          last.send(name)
+                        end
+                        arr << "#{filter_name}#{op}#{attr_example}"
+                      end
                      end.join('&')
                    else
                      'name=Joe&date>2017-01-01'
@@ -153,35 +158,33 @@ module Praxis
   
         def self.load(filters, _context = Attributor::DEFAULT_ROOT_CONTEXT, **_options)
           return filters if filters.is_a?(native_type)
-          return new if filters.nil?
-          parsed = filters.split('&').each_with_object([]) do |filter_string, arr|
-            match = FILTER_REGEX.match(filter_string)
-            values = CGI.unescape(match[:value]).split(',')
-            value = if values.size > 1
-              multimatch = true
-              values
-            else
-              multimatch = false
-              values.first
-            end
-  
-            attr_name = match[:attribute].to_sym
-            # TODO: we should coerce values if there's a mediatype defined?
-            coerced = if media_type
-              filter_components = attr_name.to_s.split('.').map(&:to_sym)
-              attr, _enclosing_type = find_filter_attribute(filter_components, media_type)
-              if multimatch
-                attr_coll = Attributor::Collection.of(attr.type)
-                attr_coll.load(value)
+          return new if filters.nil? || filters.blank?
+
+          parsed = Parser.new.parse(filters)
+          
+          tree = ConditionGroup.load(parsed)
+
+          rr = tree.flattened_conditions
+          accum = []
+          rr.each do |spec|
+            attr_name = spec[:name]
+            # TODO: Do we need to CGI.unescape things? here or even before??...
+            coerced = \
+              if media_type
+                filter_components = attr_name.to_s.split('.').map(&:to_sym)
+                attr, _enclosing_type = find_filter_attribute(filter_components, media_type)
+                if spec[:values].is_a?(Array)
+                  attr_coll = Attributor::Collection.of(attr.type)
+                  attr_coll.load(spec[:values])
+                else
+                  attr.load(spec[:values])
+                end
               else
-                attr.load(value)
+                spec[:values]
               end
-            else
-              value
-            end
-            arr.push(name: attr_name, op: match[:operator], value: coerced )
+            accum.push(name: attr_name, op: spec[:op], value: coerced , node_object: spec[:node_object])
           end
-          new(parsed)
+          new(accum)
         end
   
         def self.dump(value, **_opts)
@@ -217,21 +220,9 @@ module Praxis
               errors << "Operator #{item[:op]} not allowed for filter #{attr_name}"
             end
             value_type = attr_filters[:value_type]
-            value = item[:value]
-            if value_type && !value_type.valid_type?(value)
-              # Allow a collection of values of the right type for multimatch (if operators are = or !=)
-              if ['=','!='].include?(item[:op])
-                coll_type = Attributor::Collection.of(value_type)
-                if !coll_type.valid_type?(value)
-                  errors << "Invalid type in filter/s value for #{attr_name} " +\
-                            "(one or more of the multiple matches in #{value} are not  a #{value_type.name.split('::').last})"
-                end
-              else
-                errors << "Invalid type in filter value for #{attr_name} (#{value} using '#{item[:op]}' is not a #{value_type.name.split('::').last})"
-              end
-            end
-  
             next unless value_type == Attributor::String
+
+            value = item[:value]
             unless value.empty?
               fuzzy_match = attr_filters[:fuzzy_match]
               if (value[-1] == '*' || value[0] == '*') && !fuzzy_match