scoped_search 4.1.4 → 4.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cf2c98d3eed66905e04497673572661d98ca832a
-  data.tar.gz: 5cda23768c0ffef857dc038d74abd2539bf8881c
+SHA256:
+  metadata.gz: 9c1453ebb8fe2391f71ea35171f2115ed89bb7920a1a042d86a4393fe218888c
+  data.tar.gz: 61d2d0e948a952f7e955240747dae7e529103c5871ad3160cb2213574f0f155a
 SHA512:
-  metadata.gz: 96425ef732825dfb333d977f55a55be95ef3ac9dfda0ba9c4353fc97227d5caebfc4977b8cbdb786eb5929db84885f74435b7ca8795e449dcd2f7b432b0abbf5
-  data.tar.gz: '068d65cd7fe9e1d4e58b0080d92a22a9216ed879e17275aec5bce2651ce6cced96251803f062113c6ab66135a3ae90e478cf7b90bd2ea7b0437f5f83a96516ce'
+  metadata.gz: 04e61c564c6117d60b6f82b970df5c6c7a68362d828d89d1cf5d2bf7f6b7c9ea777cf641dbb2a21be895905305c0c7305105c9840d391aa702cd69eab706fd4d
+  data.tar.gz: 842c31b1714558d8bd4578d47b87bca4fa4298617a16168b66a77e6ff06ee1f43406b30be3557a9c50222ba5effaa9c5eec27832839e75bb11faf085f95bb7bc

data/.travis.yml

@@ -1,6 +1,9 @@
 language: ruby
 cache: bundler
 sudo: false
+services:
+  - postgresql
+  - mysql
 
 install:
 - bundle install
@@ -14,8 +17,11 @@ rvm:
 - "2.0"
 - "2.1"
 - "2.2.2"
-- "2.3.1"
+- "2.3.7"
 - "2.4.0"
+- "2.5.1"
+- "2.6.0"
+- "2.7.1"
 - ruby-head
 - jruby-19mode
 - jruby-head
@@ -24,6 +30,10 @@ gemfile:
 - Gemfile.activerecord42
 - Gemfile.activerecord50
 - Gemfile.activerecord51
+- Gemfile.activerecord52
+- Gemfile.activerecord52_with_activesupport52
+- Gemfile.activerecord60
+- Gemfile.activerecord60_with_activesupport60
 
 matrix:
   allow_failures:
@@ -39,3 +49,41 @@ matrix:
       gemfile: Gemfile.activerecord51
     - rvm: "2.1"
       gemfile: Gemfile.activerecord51
+    - rvm: "2.0"
+      gemfile: Gemfile.activerecord52
+    - rvm: "2.1"
+      gemfile: Gemfile.activerecord52
+    - rvm: "2.0"
+      gemfile: Gemfile.activerecord52_with_activesupport52
+    - rvm: "2.1"
+      gemfile: Gemfile.activerecord52_with_activesupport52
+    - rvm: "2.0"
+      gemfile: Gemfile.activerecord60
+    - rvm: "2.1"
+      gemfile: Gemfile.activerecord60
+    - rvm: "2.2.2"
+      gemfile: Gemfile.activerecord60
+    - rvm: "2.3.7"
+      gemfile: Gemfile.activerecord60
+    - rvm: "2.4.0"
+      gemfile: Gemfile.activerecord60
+    - rvm: "2.0"
+      gemfile: Gemfile.activerecord60_with_activesupport60
+    - rvm: "2.1"
+      gemfile: Gemfile.activerecord60_with_activesupport60
+    - rvm: "2.2.2"
+      gemfile: Gemfile.activerecord60_with_activesupport60
+    - rvm: "2.3.7"
+      gemfile: Gemfile.activerecord60_with_activesupport60
+    - rvm: "2.4.0"
+      gemfile: Gemfile.activerecord60_with_activesupport60
+    - rvm: "2.7.1"
+      gemfile: Gemfile.activerecord42
+    - rvm: "2.7.1"
+      gemfile: Gemfile.activerecord50
+    - rvm: "2.7.1"
+      gemfile: Gemfile.activerecord51
+    - rvm: "2.7.1"
+      gemfile: Gemfile.activerecord52
+    - rvm: "2.7.1"
+      gemfile: Gemfile.activerecord52_with_activesupport52

data/CHANGELOG.rdoc

@@ -6,7 +6,27 @@ Please add an entry to the "Unreleased changes" section in your pull requests.
 
 === Unreleased changes
 
-*Nothing yet*
+- Nothing yet
+
+=== Version 4.1.8
+
+- Fix querying in associations by set, datetime or IN searches
+- Add support for ActiveRecord 6.0 and Rails 6.1
+
+=== Version 4.1.7
+
+- When Active Support is available, we parse time respecting current time zone
+- Allow dots in rename definitions
+- Accept extra value_filter in `complete_for`
+
+=== Version 4.1.6
+
+- Support for UUID columns on PostgreSQL (#184)
+- Allow value mapping on runtime to allow queries like "user = current" (#183)
+
+=== Version 4.1.5
+
+- Bugfix related to auto-completion of virtual fields (#182)
 
 === Version 4.1.4
 

data/Gemfile.activerecord52

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'actionview', '~> 5.2.0'
+gem 'activerecord', '~> 5.2.0'
+
+platforms :jruby do
+  gem 'activerecord-jdbcsqlite3-adapter'
+  gem 'activerecord-jdbcmysql-adapter'
+  gem 'activerecord-jdbcpostgresql-adapter'
+end
+
+platforms :ruby do
+  gem 'sqlite3', '~> 1.3.6'
+  gem 'mysql2', '>= 0.3.18', '< 0.5'
+  gem 'pg', '~> 0.18'
+end

data/Gemfile.activerecord52_with_activesupport52

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'actionview', '~> 5.2.0'
+gem 'activerecord', '~> 5.2.0'
+gem 'activesupport', '~> 5.2.0'
+
+platforms :jruby do
+  gem 'activerecord-jdbcsqlite3-adapter'
+  gem 'activerecord-jdbcmysql-adapter'
+  gem 'activerecord-jdbcpostgresql-adapter'
+end
+
+platforms :ruby do
+  gem 'sqlite3', '~> 1.3.6'
+  gem 'mysql2', '>= 0.3.18', '< 0.5'
+  gem 'pg', '~> 0.18'
+end

data/Gemfile.activerecord60

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'actionview', '~> 6.0.0'
+gem 'activerecord', '~> 6.0.0'
+
+platforms :jruby do
+  gem 'activerecord-jdbcsqlite3-adapter'
+  gem 'activerecord-jdbcmysql-adapter'
+  gem 'activerecord-jdbcpostgresql-adapter'
+end
+
+platforms :ruby do
+  gem 'sqlite3', '~> 1.4'
+  gem 'mysql2', '> 0.5'
+  gem 'pg', '>= 0.18', '< 2.0'
+end

data/Gemfile.activerecord60_with_activesupport60

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'actionview', '~> 6.0.0'
+gem 'activerecord', '~> 6.0.0'
+gem 'activesupport', '~> 6.0.0'
+
+platforms :jruby do
+  gem 'activerecord-jdbcsqlite3-adapter'
+  gem 'activerecord-jdbcmysql-adapter'
+  gem 'activerecord-jdbcpostgresql-adapter'
+end
+
+platforms :ruby do
+  gem 'sqlite3', '~> 1.4'
+  gem 'mysql2', '> 0.5'
+  gem 'pg', '>= 0.18', '< 2.0'
+end

data/lib/scoped_search/auto_complete_builder.rb

@@ -201,10 +201,17 @@ module ScopedSearch
       return complete_date_value if field.temporal?
       return complete_key_value(field, token, val) if field.key_field
 
+      special_values = field.special_values.select { |v| v =~ /\A#{val}/ }
+      special_values + complete_value_from_db(field, special_values, val)
+    end
+
+    def complete_value_from_db(field, special_values, val)
+      count = 20 - special_values.count
       completer_scope(field)
+        .where(@options[:value_filter])
         .where(value_conditions(field.quoted_field, val))
         .select(field.quoted_field)
-        .limit(20)
+        .limit(count)
         .distinct
         .map(&field.field)
         .compact
@@ -213,8 +220,8 @@ module ScopedSearch
 
     def completer_scope(field)
       klass = field.klass
-      scope =  klass.respond_to?(:completer_scope) ? klass.completer_scope(@options) : klass
-      scope.respond_to?(:reorder) ? scope.reorder(field.quoted_field) : scope.scoped(:order => field.quoted_field)
+      scope = klass.respond_to?(:completer_scope) ? klass.completer_scope(@options) : klass
+      scope.respond_to?(:reorder) ? scope.reorder(Arel.sql(field.quoted_field)) : scope.scoped(:order => field.quoted_field)
     end
 
     # set value completer
@@ -270,7 +277,7 @@ module ScopedSearch
 
     # This method complete infix operators by field type
     def complete_operator(node)
-      definition.operator_by_field_name(node.value)
+      definition.operator_by_field_name(node.value).map { |o| o.end_with?(' ') ? o : "#{o} " }
     end
   end
 end

data/lib/scoped_search/definition.rb

@@ -17,7 +17,7 @@ module ScopedSearch
 
       attr_reader :definition, :field, :only_explicit, :relation, :key_relation, :full_text_search,
                   :key_field, :complete_value, :complete_enabled, :offset, :word_size, :ext_method, :operators,
-                  :validator
+                  :validator, :value_translation, :special_values
 
       # Initializes a Field instance given the definition passed to the
       # scoped_search call on the ActiveRecord-based model class.
@@ -42,7 +42,9 @@ module ScopedSearch
                      profile: nil,
                      relation: nil,
                      rename: nil,
+                     special_values: [],
                      validator: nil,
+                     value_translation: nil,
                      word_size: 1,
                      **kwargs)
 
@@ -50,6 +52,8 @@ module ScopedSearch
         raise ArgumentError, "Missing field or 'on' keyword argument" if on.nil?
         @field = on.to_sym
 
+        raise ArgumentError, "'special_values' must be an Array" unless special_values.kind_of?(Array)
+
         # Reserved Ruby keywords so access via kwargs instead, but deprecate them for future versions
         if kwargs.key?(:in)
           relation = kwargs.delete(:in)
@@ -77,8 +81,10 @@ module ScopedSearch
         @only_explicit    = !!only_explicit
         @operators        = operators
         @relation         = relation
+        @special_values   = special_values
         @validator        = validator
         @word_size        = word_size
+        @value_translation = value_translation
 
         # Store this field in the field array
         definition.define_field(rename || @field, self)
@@ -107,6 +113,11 @@ module ScopedSearch
         end
       end
 
+      # Returns true if this is a virtual field.
+      def virtual?
+        !ext_method.nil?
+      end
+
       # Returns the ActiveRecord column definition that corresponds to this field.
       def column
         @column ||= begin
@@ -120,15 +131,15 @@ module ScopedSearch
 
       # Returns the column type of this field.
       def type
-        @type ||= column.type
+        @type ||= virtual? ? :virtual : column.type
       end
 
-      # Returns true if this field is a datetime-like column
+      # Returns true if this field is a datetime-like column.
       def datetime?
         [:datetime, :time, :timestamp].include?(type)
       end
 
-      # Returns true if this field is a date-like column
+      # Returns true if this field is a date-like column.
       def date?
         type == :date
       end
@@ -149,6 +160,10 @@ module ScopedSearch
         [:string, :text].include?(type)
       end
 
+      def uuid?
+        type == :uuid
+      end
+
       # Returns true if this is a set.
       def set?
         complete_value.is_a?(Hash)
@@ -167,7 +182,7 @@ module ScopedSearch
         return "#{field} #{order}"
       end
 
-      # Return 'table'.'column' with the correct database quotes
+      # Return 'table'.'column' with the correct database quotes.
       def quoted_field
         c = klass.connection
         "#{c.quote_table_name(klass.table_name)}.#{c.quote_column_name(field)}"
@@ -223,6 +238,9 @@ module ScopedSearch
       if field.nil?
         dotted = name.to_s.split('.')[0]
         field = fields[dotted.to_sym] unless dotted.blank?
+        if field && field.key_relation.nil?
+          return nil
+        end
       end
       field
     end
@@ -231,24 +249,27 @@ module ScopedSearch
     def operator_by_field_name(name)
       field = field_by_name(name)
       return [] if field.nil?
-      return field.operators                                      if field.operators
-      return ['= ', '!= ']                                        if field.set?
-      return ['= ', '> ', '< ', '<= ', '>= ','!= ', '^ ', '!^ ']  if field.numerical?
-      return ['= ', '!= ', '~ ', '!~ ', '^ ', '!^ ']              if field.textual?
-      return ['= ', '> ', '< ']                                   if field.temporal?
+      return field.operators                                          if field.operators
+      return ['=', '!=', '>', '<', '<=', '>=', '~', '!~', '^', '!^']  if field.virtual?
+      return ['=', '!=']                                              if field.set? || field.uuid?
+      return ['=', '>', '<', '<=', '>=', '!=', '^', '!^']             if field.numerical?
+      return ['=', '!=', '~', '!~', '^', '!^']                        if field.textual?
+      return ['=', '>', '<']                                          if field.temporal?
       raise ScopedSearch::QueryNotSupported, "Unsupported type '#{field.type.inspect}')' for field '#{name}'. This can be a result of a search definition problem."
     end
 
     NUMERICAL_REGXP = /^\-?\d+(\.\d+)?$/
     INTEGER_REGXP = /^\-?\d+$/
+    UUID_REGXP = /^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/
 
     # Returns a list of appropriate fields to search in given a search keyword and operator.
     def default_fields_for(value, operator = nil)
 
-      column_types  = []
+      column_types  = [:virtual]
       column_types += [:string, :text]                if [nil, :like, :unlike, :ne, :eq].include?(operator)
       column_types += [:double, :float, :decimal]     if value =~ NUMERICAL_REGXP
       column_types += [:integer]                      if value =~ INTEGER_REGXP
+      column_types += [:uuid]                         if value =~ UUID_REGXP
       column_types += [:datetime, :date, :timestamp]  if (parse_temporal(value))
 
       default_fields.select { |field| !field.set? && column_types.include?(field.type) }
@@ -257,6 +278,8 @@ module ScopedSearch
     # Try to parse a string as a datetime.
     # Supported formats are Today, Yesterday, Sunday, '1 day ago', '2 hours ago', '3 months ago', '4 weeks from now', 'Jan 23, 2004'
     # And many more formats that are documented in Ruby DateTime API Doc.
+    # In case Time responds to #zone, we know this is Rails environment and we can use Time.zone.parse. The benefit is that the
+    # current timezone is respected and does not have to be specified explicitly. That way even relative dates work as expected.
     def parse_temporal(value)
       return Date.current if value =~ /\btoday\b/i
       return 1.day.ago.to_date if value =~ /\byesterday\b/i
@@ -265,7 +288,12 @@ module ScopedSearch
       return (eval($1.strip.gsub(/\s+/,'.').downcase)).to_date     if value =~ /\A\s*(\d+\s+\b(?:days?|weeks?|months?|years?)\b\s+\bago)\b\s*\z/i
       return (eval($1.strip.gsub(/from\s+now/i,'from_now').gsub(/\s+/,'.').downcase)).to_datetime if value =~ /\A\s*(\d+\s+\b(?:hours?|minutes?)\b\s+\bfrom\s+now)\b\s*\z/i
       return (eval($1.strip.gsub(/from\s+now/i,'from_now').gsub(/\s+/,'.').downcase)).to_date     if value =~ /\A\s*(\d+\s+\b(?:days?|weeks?|months?|years?)\b\s+\bfrom\s+now)\b\s*\z/i
-      DateTime.parse(value, true) rescue nil
+      if Time.respond_to?(:zone) && !Time.zone.nil?
+        parsed = Time.zone.parse(value) rescue nil
+        parsed && parsed.to_datetime
+      else
+        DateTime.parse(value, true) rescue nil
+      end
     end
 
     # Returns a list of fields that should be searched on by default.
@@ -298,11 +326,11 @@ module ScopedSearch
 
         search_scope = klass.all
         find_options = ScopedSearch::QueryBuilder.build_query(definition, query || '', options)
-        search_scope = search_scope.where(find_options[:conditions])   if find_options[:conditions]
-        search_scope = search_scope.includes(find_options[:include])   if find_options[:include]
-        search_scope = search_scope.joins(find_options[:joins])        if find_options[:joins]
-        search_scope = search_scope.reorder(find_options[:order])      if find_options[:order]
-        search_scope = search_scope.references(find_options[:include]) if find_options[:include]
+        search_scope = search_scope.where(find_options[:conditions])        if find_options[:conditions]
+        search_scope = search_scope.includes(find_options[:include])        if find_options[:include]
+        search_scope = search_scope.joins(find_options[:joins])             if find_options[:joins]
+        search_scope = search_scope.reorder(Arel.sql(find_options[:order])) if find_options[:order]
+        search_scope = search_scope.references(find_options[:include])      if find_options[:include]
 
         search_scope
       end

data/lib/scoped_search/query_builder.rb

@@ -113,7 +113,7 @@ module ScopedSearch
     # By default, it will simply look up the correct SQL operator in the SQL_OPERATORS
     # hash, but this can be overridden by a database adapter.
     def sql_operator(operator, field)
-      raise ScopedSearch::QueryNotSupported, "the operator '#{operator}' is not supported for field type '#{field.type}'" if !field.ext_method and [:like, :unlike].include?(operator) and !field.textual?
+      raise ScopedSearch::QueryNotSupported, "the operator '#{operator}' is not supported for field type '#{field.type}'" if !field.virtual? and [:like, :unlike].include?(operator) and !field.textual?
       SQL_OPERATORS[operator]
     end
 
@@ -137,7 +137,7 @@ module ScopedSearch
 
       # Parse the value as a date/time and ignore invalid timestamps
       timestamp = definition.parse_temporal(value)
-      return nil unless timestamp
+      return [] unless timestamp
 
       timestamp = timestamp.to_date if field.date?
       # Check for the case that a date-only value is given as search keyword,
@@ -149,11 +149,9 @@ module ScopedSearch
         if [:eq, :ne].include?(operator)
           # Instead of looking for an exact (non-)match, look for dates that
           # fall inside/outside the range of timestamps of that day.
-          yield(:parameter, timestamp)
-          yield(:parameter, timestamp + span)
           negate    = (operator == :ne) ? 'NOT ' : ''
           field_sql = field.to_sql(operator, &block)
-          return "#{negate}(#{field_sql} >= ? AND #{field_sql} < ?)"
+          return ["#{negate}(#{field_sql} >= ? AND #{field_sql} < ?)", timestamp, timestamp + span]
 
         elsif operator == :gt
           # Make sure timestamps on the given date are not included in the results
@@ -169,9 +167,8 @@ module ScopedSearch
         end
       end
 
-      # Yield the timestamp and return the SQL test
-      yield(:parameter, timestamp)
-      "#{field.to_sql(operator, &block)} #{sql_operator(operator, field)} ?"
+      # return the SQL test
+      ["#{field.to_sql(operator, &block)} #{sql_operator(operator, field)} ?", timestamp]
     end
 
     # Validate the key name is in the set and translate the value to the set value.
@@ -181,6 +178,14 @@ module ScopedSearch
       translated_value
     end
 
+    def map_value(field, value)
+      old_value = value
+      translator = field.value_translation
+      value = translator.call(value) if translator
+      raise ScopedSearch::QueryNotSupported, "Translation from any value to nil is not allowed, translated '#{old_value}'" if value.nil?
+      value
+    end
+
     # A 'set' is group of possible values, for example a status might be "on", "off" or "unknown" and the database representation
     # could be for example a numeric value. This method will validate the input and translate it into the database representation.
     def set_test(field, operator,value, &block)
@@ -197,8 +202,7 @@ module ScopedSearch
           set_value = false
         end
       end
-      yield(:parameter, set_value)
-      return "#{negate}(#{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} ?)"
+      ["#{negate}(#{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} ?)", set_value]
     end
 
     # Generates a simple SQL test expression, for a field and value using an operator.
@@ -210,43 +214,51 @@ module ScopedSearch
     # <tt>operator</tt>:: The operator used for comparison.
     # <tt>value</tt>:: The value to compare the field with.
     def sql_test(field, operator, value, lhs, &block) # :yields: finder_option_type, value
-      return field.to_ext_method_sql(lhs, sql_operator(operator, field), value, &block) if field.ext_method
+      return field.to_ext_method_sql(lhs, sql_operator(operator, field), value, &block) if field.virtual?
 
       yield(:keyparameter, lhs.sub(/^.*\./,'')) if field.key_field
 
-      if [:like, :unlike].include?(operator)
-        yield(:parameter, (value !~ /^\%|\*/ && value !~ /\%|\*$/) ? "%#{value}%" : value.tr_s('%*', '%'))
-        return "#{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} ?"
-
-      elsif [:in, :notin].include?(operator)
-        value.split(',').collect { |v| yield(:parameter, field.set? ? translate_value(field, v) : v.strip) }
-        value = value.split(',').collect { "?" }.join(",")
-        return "#{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} (#{value})"
+      condition, *values = if field.temporal?
+                             datetime_test(field, operator, value, &block)
+                           elsif field.set?
+                             set_test(field, operator, value, &block)
+                           else
+                             ["#{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} #{value_placeholders(operator, value)}", value]
+                           end
+      values.each { |value| preprocess_parameters(field, operator, value, &block) }
 
-      elsif field.temporal?
-        return datetime_test(field, operator, value, &block)
+      if field.relation && definition.reflection_by_name(field.definition.klass, field.relation).macro == :has_many
+        connection = field.definition.klass.connection
+        reflection = definition.reflection_by_name(field.definition.klass, field.relation)
+        primary_key_col = reflection.options[:primary_key] || field.definition.klass.primary_key
+        primary_key = "#{connection.quote_table_name(field.definition.klass.table_name)}.#{connection.quote_column_name(primary_key_col)}"
+        key, join_table = if reflection.options.has_key?(:through)
+                            [primary_key, has_many_through_join(field)]
+                          else
+                            [connection.quote_column_name(field.reflection_keys(reflection)[1]),
+                             connection.quote_table_name(field.klass.table_name)]
+                          end
+
+        condition = "#{primary_key} IN (SELECT #{key} FROM #{join_table} WHERE #{condition} )"
+      end
+      condition
+    end
 
-      elsif field.set?
-        return set_test(field, operator, value, &block)
+    def preprocess_parameters(field, operator, value, &block)
+      values = if [:in, :notin].include?(operator)
+                 value.split(',').map { |v| map_value(field, field.set? ? translate_value(field, v) : v.strip) }
+               elsif [:like, :unlike].include?(operator)
+                 [(value !~ /^\%|\*/ && value !~ /\%|\*$/) ? "%#{value}%" : value.tr_s('%*', '%')]
+               else
+                 [map_value(field, field.offset ? value.to_i : value)]
+               end
+      values.each { |value| yield(:parameter, value) }
+    end
 
-      elsif field.relation && definition.reflection_by_name(field.definition.klass, field.relation).macro == :has_many
-        value = value.to_i if field.offset
-        yield(:parameter, value)
-        connection = field.definition.klass.connection
-        primary_key = "#{connection.quote_table_name(field.definition.klass.table_name)}.#{connection.quote_column_name(field.definition.klass.primary_key)}"
-        if definition.reflection_by_name(field.definition.klass, field.relation).options.has_key?(:through)
-          join = has_many_through_join(field)
-          return "#{primary_key} IN (SELECT #{primary_key} FROM #{join} WHERE #{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} ? )"
-        else
-          foreign_key = connection.quote_column_name(field.reflection_keys(definition.reflection_by_name(field.definition.klass, field.relation))[1])
-          return "#{primary_key} IN (SELECT #{foreign_key} FROM #{connection.quote_table_name(field.klass.table_name)} WHERE #{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} ? )"
-        end
+    def value_placeholders(operator, value)
+      return '?' unless [:in, :notin].include?(operator)
 
-      else
-        value = value.to_i if field.offset
-        yield(:parameter, value)
-        return "#{field.to_sql(operator, &block)} #{self.sql_operator(operator, field)} ?"
-      end
+      '(' + value.split(',').map { '?' }.join(',') + ')'
     end
 
     def find_has_many_through_association(field, through)
@@ -259,46 +271,46 @@ module ScopedSearch
       middle_table_association
     end
 
+    # Walk the chain of has-many-throughs, collecting all tables we will need to join
+    def nested_has_many(many_class, relation)
+      acc = [relation]
+      while (reflection = definition.reflection_by_name(many_class, relation))
+        break if reflection.nil? || reflection.options[:through].nil?
+        relation = reflection.options[:through]
+        acc.unshift(relation)
+      end
+      acc.map { |relation| definition.reflection_by_name(many_class, relation) }
+    end
+
     def has_many_through_join(field)
       many_class = field.definition.klass
-      through = definition.reflection_by_name(many_class, field.relation).options[:through]
-      through_class = definition.reflection_by_name(many_class, through).klass
-
       connection = many_class.connection
-
-      # table names
-      endpoint_table_name = field.klass.table_name
-      many_table_name = many_class.table_name
-      middle_table_name = through_class.table_name
-
-      # primary and foreign keys + optional conditions for the joins
-      pk1, fk1   = field.reflection_keys(definition.reflection_by_name(many_class, through))
-      condition_many_to_middle = if with_polymorphism?(many_class, field.klass, through, through_class)
-                                   field.reflection_conditions(definition.reflection_by_name(field.klass, many_table_name))
-                                 else
-                                   ''
-                                 end
-      condition_middle_to_end = field.reflection_conditions(definition.reflection_by_name(field.klass, middle_table_name))
-
-      # primary and foreign keys + optional condition for the endpoint to middle join
-      middle_table_association = find_has_many_through_association(field, through) || middle_table_name
-      pk2, fk2   = field.reflection_keys(definition.reflection_by_name(field.klass, middle_table_association))
-      condition2 = field.reflection_conditions(definition.reflection_by_name(many_class, field.relation))
-
-      <<-SQL
-        #{connection.quote_table_name(many_table_name)}
-        INNER JOIN #{connection.quote_table_name(middle_table_name)}
-        ON #{connection.quote_table_name(many_table_name)}.#{connection.quote_column_name(pk1)} = #{connection.quote_table_name(middle_table_name)}.#{connection.quote_column_name(fk1)} #{condition_many_to_middle} #{condition_middle_to_end}
-        INNER JOIN #{connection.quote_table_name(endpoint_table_name)}
-        ON #{connection.quote_table_name(middle_table_name)}.#{connection.quote_column_name(fk2)} = #{connection.quote_table_name(endpoint_table_name)}.#{connection.quote_column_name(pk2)} #{condition2}
-      SQL
+      sql = connection.quote_table_name(many_class.table_name)
+      join_reflections = nested_has_many(many_class, field.relation)
+      table_names = [many_class.table_name] + join_reflections.map(&:table_name)
+
+      join_reflections.zip(table_names.zip(join_reflections.drop(1))).reduce(sql) do |acc, (reflection, (previous_table, next_reflection))|
+        klass = reflection.method(:join_keys).arity == 1 ? [reflection.klass] : [] # ActiveRecord <5.2 workaround
+        fk1, pk1 = reflection.join_keys(*klass).values # We are joining the tables "in reverse", so the PK and FK are swapped
+
+        # primary and foreign keys + optional conditions for the joins
+        join_condition = if with_polymorphism?(reflection)
+                           field.reflection_conditions(definition.reflection_by_name(next_reflection.klass, previous_table))
+                         else
+                           ''
+                         end
+
+        acc + <<-SQL
+          INNER JOIN #{connection.quote_table_name(reflection.table_name)}
+          ON #{connection.quote_table_name(previous_table)}.#{connection.quote_column_name(pk1)} = #{connection.quote_table_name(reflection.table_name)}.#{connection.quote_column_name(fk1)} #{join_condition}
+        SQL
+      end
     end
 
-    def with_polymorphism?(many_class, endpoint_class, through, through_class)
-      reflections = [definition.reflection_by_name(endpoint_class, through), definition.reflection_by_name(many_class, through)].compact
-      as = reflections.map(&:options).compact.map { |opt| opt[:as] }.compact
-      return false if as.empty?
-      definition.reflection_by_name(through_class, as.first).options[:polymorphic]
+    def with_polymorphism?(reflection)
+      as = reflection.options[:as]
+      return unless as
+      definition.reflection_by_name(reflection.klass, as).options[:polymorphic]
     end
 
     # This module gets included into the Field class to add SQL generation.
@@ -403,7 +415,7 @@ module ScopedSearch
       def to_ext_method_sql(key, operator, value, &block)
         raise ScopedSearch::QueryNotSupported, "'#{definition.klass}' doesn't respond to '#{ext_method}'" unless definition.klass.respond_to?(ext_method)
         begin
-          conditions = definition.klass.send(ext_method.to_sym, key, operator, value) 
+          conditions = definition.klass.send(ext_method.to_sym, key, operator, value)
         rescue StandardError => e
           raise ScopedSearch::QueryNotSupported, "external method '#{ext_method}' failed with error: #{e}"
         end
@@ -517,7 +529,7 @@ module ScopedSearch
         def validate_value(field, value)
           validator = field.validator
           if validator
-            valid = validator.call(value)
+            valid = field.special_values.include?(value) || validator.call(value)
             raise ScopedSearch::QueryNotSupported, "Value '#{value}' is not valid for field '#{field.field}'" unless valid
           end
         end
@@ -554,7 +566,7 @@ module ScopedSearch
       # Switches out the default LIKE operator in the default <tt>sql_operator</tt>
       # method for ILIKE or @@ if full text searching is enabled.
       def sql_operator(operator, field)
-        raise ScopedSearch::QueryNotSupported, "the operator '#{operator}' is not supported for field type '#{field.type}'" if !field.ext_method and [:like, :unlike].include?(operator) and !field.textual?
+        raise ScopedSearch::QueryNotSupported, "the operator '#{operator}' is not supported for field type '#{field.type}'" if !field.virtual? and [:like, :unlike].include?(operator) and !field.textual?
         return '@@' if [:like, :unlike].include?(operator) && field.full_text_search
         case operator
           when :like   then 'ILIKE'