mincer 0.1.2 → 0.2.0

data/lib/mincer/processors/pg_search/processor.rb

@@ -0,0 +1,148 @@
+# This and all processors are heavily influenced by pg_search(https://github.com/Casecommons/pg_search)
+module Mincer
+  module Processors
+    module PgSearch
+      class Processor
+        include ::Mincer::Processors::Helpers
+
+        def initialize(mincer)
+          @mincer, @args, @relation = mincer, mincer.args, mincer.relation
+        end
+
+        def apply
+          if Mincer.postgres?
+            @relation = apply_pg_search(@relation, @args)
+          else
+            @relation
+          end
+        end
+
+        def apply_pg_search(relation, args)
+          relation.where(conditions(args)).reorder(rank(args))
+        end
+
+        def conditions(args)
+          search_statements_conditions = search_statements.map do |search_statement|
+            conditions = pg_search_engines(args, search_statement).map do |pg_search_engine|
+              pg_search_engine.conditions
+            end.compact
+            join_expressions(conditions, options[:join_with] || :or)
+          end.compact
+          join_expressions(search_statements_conditions, options[:join_with] || :or).try(:to_sql)
+        end
+
+        def rank(args)
+          search_statements_conditions = search_statements.map do |search_statement|
+            conditions = pg_search_engines(args, search_statement).map do |pg_search_engine|
+              pg_search_engine.rank
+            end.compact
+            join_expressions(conditions, :+)
+          end.compact
+          rank = join_expressions(search_statements_conditions, :+).try(:to_sql)
+          "#{rank} DESC" if rank.present?
+        end
+
+        def pg_search_engines(args, search_statement)
+          Mincer.config.pg_search.engines.map do |engine_class|
+            engine_class.new(args, [search_statement])
+          end
+        end
+
+        def search_statements
+          @search_statements ||= params.any? { |param| param[:columns] } ? search_statements_from_params : default_search_statements
+        end
+
+        def search_statements_from_params
+          params.map do |param|
+            par = param.dup
+            SearchStatement.new(par.delete(:columns), search_statement_default_options(param[:engines]).merge(par))
+          end
+        end
+
+        # We use only text/string columns and avoid array
+        def default_search_statements
+          column_names = @relation.columns.reject do |column|
+            ![:string, :text].include?(column.type) || column.array
+          end.map do |column|
+            "#{@relation.table_name}.#{column.name}"
+          end
+          [SearchStatement.new(column_names, search_statement_default_options([:fulltext]).merge(engines: [:fulltext]))]
+        end
+
+        def params
+          Array.wrap(@mincer.send(:pg_search_params))
+        end
+
+        def options
+          @mincer.send(:pg_search_options)
+        end
+
+        def search_statement_default_options(engines)
+          (engines & [:fulltext, :trigram, :array]).inject({}) do |options, engine|
+            options = Mincer.config.pg_search.send("#{engine}_engine").merge(options)
+            options
+          end
+        end
+
+      end
+
+      module Options
+        extend ActiveSupport::Concern
+
+        def sort_by_rank
+        end
+
+        module ClassMethods
+          def skip_pg_search!
+            active_processors.delete(Mincer::Processors::PgSearch::Processor)
+          end
+
+          def skip_search!
+            skip_pg_search!
+          end
+
+          def pg_search(params, options = {})
+            class_eval <<-OPTIONS, __FILE__, __LINE__
+            def pg_search_params
+              @pg_search_params ||= #{params.inspect}
+                end
+            def pg_search_options
+              @pg_search_options ||= #{options.inspect}
+                end
+            OPTIONS
+          end
+        end
+
+        def pg_search_params
+          @pg_search_params ||= {}
+        end
+
+        def pg_search_options
+          @pg_search_options ||= {}
+        end
+      end
+
+      class Configuration
+        include ActiveSupport::Configurable
+        config_accessor :param_name do
+          'pattern'
+        end
+        config_accessor :fulltext_engine do
+          { ignore_accent: true, any_word: false, dictionary: :simple, ignore_case: false }
+        end
+        config_accessor :trigram_engine do
+          { ignore_accent: true, threshold: 0.3 }
+        end
+        config_accessor :array_engine do
+          { ignore_accent: true, any_word: true }
+        end
+        config_accessor :engines do
+          [Mincer::PgSearch::SearchEngines::Fulltext, Mincer::PgSearch::SearchEngines::Array, Mincer::PgSearch::SearchEngines::Trigram]
+        end
+      end
+
+    end
+  end
+end
+
+::Mincer.add_processor(:pg_search)

data/lib/mincer/processors/pg_search/sanitizer.rb

@@ -0,0 +1,61 @@
+module Mincer
+  module Processors
+    module PgSearch
+      class Sanitizer
+        AVAILABLE_SANITIZERS = [:coalesce, :ignore_case, :ignore_accent]
+        attr_accessor :term, :sanitizers
+
+        def initialize(term, *sanitizers)
+          @term, @sanitizers = term, AVAILABLE_SANITIZERS & Array.wrap(sanitizers).flatten
+        end
+
+        def sanitize_column
+          @sanitized_column ||= sanitize(Arel.sql(@term))
+        end
+
+        def sanitize_string(options = {})
+          if sanitizers.empty?
+            return options[:quote] ? Mincer.connection.quote(@term) : @term
+          end
+          @sanitized_string ||= sanitize(@term)
+        end
+
+        def sanitize(node)
+          sanitizers.inject(node) do |query, sanitizer|
+            query = self.class.send(sanitizer, query)
+            query
+          end
+        end
+
+        def self.sanitize_column(term, *sanitizers)
+          new(term, *sanitizers).sanitize_column
+        end
+
+        def self.sanitize_string(term, *sanitizers)
+          new(term, *sanitizers).sanitize_string
+        end
+
+        def self.sanitize_string_quoted(term, *sanitizers)
+          new(term, *sanitizers).sanitize_string(quote: true)
+        end
+
+        def self.ignore_case(term)
+          Arel::Nodes::NamedFunction.new('lower', [term])
+        end
+
+        def self.ignore_accent(term)
+          Arel::Nodes::NamedFunction.new('unaccent', [term])
+        end
+
+        def self.coalesce(term, val = '')
+          if Mincer.pg_extension_installed?(:unaccent)
+            Arel::Nodes::NamedFunction.new('coalesce', [term, val])
+          else
+            term
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/mincer/processors/pg_search/search_engines/array.rb

@@ -0,0 +1,41 @@
+module Mincer
+  module PgSearch
+    module SearchEngines
+      class Array < Base
+
+        def conditions
+          return nil unless prepared_search_statements.any?
+          arel_group do
+            conditions = prepared_search_statements.map do |search_statement|
+              if search_statement.pattern = args[search_statement.param_name]
+                terms_delimiter = search_statement.options[:any_word] ? '&&' : '@>'
+                arel_group(Arel::Nodes::InfixOperation.new(terms_delimiter, document_for(search_statement), query_for(search_statement)))
+              end
+            end
+            join_expressions(conditions, :or)
+          end
+        end
+
+        private
+
+        def document_for(search_statement)
+          arel_group do
+            documents = search_statement.columns.map do |search_column|
+              Arel.sql(search_column + '::text[]')
+            end
+            join_expressions(documents, '||')
+          end
+        end
+
+        def query_for(search_statement)
+          normalized_pattern = search_statement.pattern.split(%r{\s|,}).uniq.reject(&:empty?).map do |item|
+            sanitize_string_quoted(item, search_statement.sanitizers).to_sql
+          end.join(',')
+          Arel.sql("ARRAY[#{normalized_pattern}]")
+        end
+
+      end
+
+    end
+  end
+end

data/lib/mincer/processors/pg_search/search_engines/base.rb

@@ -0,0 +1,56 @@
+module Mincer
+  module PgSearch
+    module SearchEngines
+      class Base
+        include ::Mincer::Processors::Helpers
+        attr_reader :args, :search_statements
+
+        def initialize(args, search_statements)
+          @args, @search_statements = ::ActiveSupport::HashWithIndifferentAccess.new(args), search_statements
+        end
+
+        def arel_group(sql_string = nil)
+          sql_string = yield if block_given?
+          arel_query = sql_string.is_a?(String) ? Arel.sql(sql_string) : sql_string
+          Arel::Nodes::Grouping.new(arel_query)
+        end
+
+        def sanitize_column(term, sanitizers)
+          ::Mincer::Processors::PgSearch::Sanitizer.sanitize_column(term, sanitizers)
+        end
+
+        def sanitize_string(term, sanitizers)
+          ::Mincer::Processors::PgSearch::Sanitizer.sanitize_string(term, sanitizers)
+        end
+
+        def sanitize_string_quoted(term, sanitizers)
+          ::Mincer::Processors::PgSearch::Sanitizer.sanitize_string_quoted(term, sanitizers)
+        end
+
+        def search_engine_statements
+          @search_engine_statements ||= self.search_statements.select do |search_statement|
+            search_statement.options[:engines].try(:include?, engine_sym)
+          end
+        end
+
+        def prepared_search_statements
+          @prepared_search_statements ||= search_engine_statements.map do |search_statement|
+            search_statement.pattern = args[search_statement.param_name]
+            search_statement.pattern.present? ? search_statement : nil
+          end.compact
+        end
+
+        # Redefine this method in subclass if your engine name does not match class
+        def engine_sym
+          @engine_sym ||= self.class.name.to_s.demodulize.underscore.to_sym
+        end
+
+        def rank
+          #Must be implemented in subclasses
+          nil
+        end
+
+      end
+    end
+  end
+end

data/lib/mincer/processors/pg_search/search_engines/fulltext.rb

@@ -0,0 +1,58 @@
+module Mincer
+  module PgSearch
+    module SearchEngines
+      class Fulltext < Base
+        DISALLOWED_TSQUERY_CHARACTERS = /[!(:&|)'?\\]/
+
+        def conditions
+          return nil unless prepared_search_statements.any?
+          arel_group do
+            conditions = prepared_search_statements.map do |search_statement|
+              arel_group(Arel::Nodes::InfixOperation.new('@@', document_for(search_statement), query_for(search_statement)))
+            end
+            join_expressions(conditions, :or)
+          end
+        end
+
+        def rank
+          return nil unless prepared_search_statements.any?
+          arel_group do
+            ranks = prepared_search_statements.map do |search_statement|
+              Arel::Nodes::NamedFunction.new('ts_rank', [document_for(search_statement), query_for(search_statement)]).to_sql
+            end
+            join_expressions(ranks, '+')
+          end
+        end
+
+        private
+
+        def prepared_search_statements
+          @prepared_search_statements ||= search_engine_statements.map do |search_statement|
+            pattern = args[search_statement.param_name]
+            search_statement.pattern = pattern && pattern.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ').split(' ').compact
+            search_statement.pattern.present? && search_statement.pattern.any? ? search_statement : nil
+          end.compact
+        end
+
+        def document_for(search_statement)
+          arel_group do
+            search_statement.columns.map do |search_column|
+              sanitized_term = sanitize_column(search_column, search_statement.sanitizers + [:coalesce])
+              Arel::Nodes::NamedFunction.new('to_tsvector', [search_statement.dictionary, sanitized_term]).to_sql
+            end.join(' || ')
+          end
+        end
+
+        def query_for(search_statement)
+          terms_delimiter = search_statement.options[:any_word] ? '|' : '&'
+          tsquery_sql = Arel.sql(search_statement.terms.map { |term| sanitize_string_quoted(term, search_statement.sanitizers).to_sql }.join(" || ' #{terms_delimiter} ' || "))
+          arel_group do
+            Arel::Nodes::NamedFunction.new('to_tsquery', [search_statement.dictionary, tsquery_sql])
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/mincer/processors/pg_search/search_engines/trigram.rb

@@ -0,0 +1,41 @@
+module Mincer
+  module PgSearch
+    module SearchEngines
+      class Trigram < Base
+
+        def conditions
+          return nil unless prepared_search_statements.any?
+          arel_group do
+            join_expressions(prepared_search_statements.map { |search_statement| document_for(search_statement) }, :or)
+          end
+        end
+
+        def rank
+          return nil unless prepared_search_statements.any?
+          arel_group do
+            join_expressions(prepared_search_statements.map { |search_statement| rank_for(search_statement) }, :+)
+          end
+        end
+
+        private
+
+
+        def document_for(search_statement)
+          documents = search_statement.columns.map do |search_column|
+            similarity = Arel::Nodes::NamedFunction.new('similarity', [sanitize_column(search_column, search_statement.sanitizers), sanitize_string(search_statement.pattern, search_statement.sanitizers)])
+            arel_group(similarity.gteq(search_statement.threshold))
+          end
+          join_expressions(documents, :or)
+        end
+
+        def rank_for(search_statement)
+          ranks = search_statement.columns.map do |search_column|
+            Arel::Nodes::NamedFunction.new('similarity', [sanitize_column(search_column, search_statement.sanitizers), sanitize_string(search_statement.pattern, search_statement.sanitizers)])
+          end
+          join_expressions(ranks, :+)
+        end
+
+      end
+    end
+  end
+end

data/lib/mincer/processors/pg_search/search_statement.rb

@@ -0,0 +1,31 @@
+module Mincer
+  module Processors
+    module PgSearch
+      class SearchStatement
+        attr_accessor :columns, :options, :pattern
+        alias_method :terms, :pattern
+
+        def initialize(columns, options = {})
+          @columns, @options = columns, ::ActiveSupport::HashWithIndifferentAccess.new(options)
+        end
+
+        def sanitizers
+          @sanitizers ||= Sanitizer::AVAILABLE_SANITIZERS.select { |sanitizer| options[sanitizer] }
+        end
+
+        def dictionary
+          options[:dictionary] || Mincer.config.pg_search.fulltext_engine[:dictionary]
+        end
+
+        def threshold
+          options[:threshold] || Mincer.config.pg_search.trigram_engine[:threshold]
+        end
+
+        def param_name
+          options[:param_name] || Mincer.config.pg_search.param_name
+        end
+
+      end
+    end
+  end
+end

data/lib/mincer/processors/sorting/processor.rb

@@ -0,0 +1,113 @@
+module Mincer
+  module Processors
+    module Sorting
+      class Processor
+        def initialize(mincer)
+          @mincer, @args, @relation = mincer, mincer.args, mincer.relation
+        end
+
+        def apply
+          relation = @relation.order(sort_string)
+          @mincer.sort_attribute, @mincer.sort_order = sort_attr, order_attr
+          relation
+        end
+
+        def sort_string
+          if sort_attr
+            "#{sort_attr} #{order_attr || default_order}"
+          else
+            "#{default_sort} #{order_attr || default_order}"
+          end
+        end
+
+        def sort_attr
+          @mincer.send(:allowed_sort_attributes).include?(sort) && sort
+        end
+
+        def order_attr
+          %w{asc desc}.include?(order.try(:downcase)) && order
+        end
+
+        def sort
+          @args[::Mincer.config.sorting.sort_param_name]
+        end
+
+        def default_sort
+          @mincer.try(:default_sort_attribute)
+        end
+
+        def order
+          @args[::Mincer.config.sorting.order_param_name]
+        end
+
+        def default_order
+          @mincer.try(:default_sort_order)
+        end
+
+      end
+
+
+      module Options
+        extend ActiveSupport::Concern
+
+        included do
+          # Used in view helpers
+          attr_accessor :sort_attribute, :sort_order
+        end
+
+        module ClassMethods
+          def skip_sorting!
+            active_processors.delete(Mincer::Processors::Sorting::Processor)
+          end
+        end
+
+        # Default sort attribute. You must override this method if you want something else
+        def default_sort_attribute
+          ::Mincer.config.sorting.sort_attribute
+        end
+
+        # Default order attribute. You must override this method if you want something else
+        def default_sort_order
+          ::Mincer.config.sorting.order_attribute
+        end
+
+        # Allowed sort attributes, should return array of strings
+        def allowed_sort_attributes
+          @scope.attribute_names
+        end
+      end
+
+      class Configuration
+        include ActiveSupport::Configurable
+
+        config_accessor :sort_param_name do
+          :sort
+        end
+
+        config_accessor :sort_attribute do
+          :id
+        end
+
+        config_accessor :order_param_name do
+          :order
+        end
+
+        config_accessor :order_attribute do
+          :asc
+        end
+
+        config_accessor :asc_class do
+          'sorted order_down'
+        end
+
+        config_accessor :desc_class do
+          'sorted order_up'
+        end
+
+      end
+
+    end
+  end
+end
+
+::Mincer.add_processor(:sorting)

data/lib/mincer/version.rb

@@ -1,7 +1,7 @@
 module Mincer
 
   def self.version
-    Gem::Version.new '0.1.2'
+    Gem::Version.new '0.2.0'
   end
 
   module VERSION #:nodoc:

data/lib/mincer.rb

@@ -1,6 +1,7 @@
+require 'ostruct'
 require 'mincer/version'
-
-
+require 'mincer/core_ext/string'
+require 'mincer/config'
 require 'mincer/base'
 
 module Mincer
@@ -15,24 +16,39 @@ module Mincer
   def self.connection
     ::ActiveRecord::Base.connection()
   end
-end
 
+  def self.add_processor(processor)
+    processor_scope = ::Mincer::Processors.const_get(ActiveSupport::Inflector.camelize(processor.to_s, true))
+    ::Mincer.processors << processor_scope.const_get('Processor')
+    ::Mincer::Base.send(:include, processor_scope.const_get('Options')) if processor_scope.const_defined?('Options')
+    ::Mincer.config.add(processor, processor_scope.const_get('Configuration'))  if processor_scope.const_defined?('Configuration') if processor_scope.const_defined?('Configuration')
+  end
 
-# Loading processors
-require 'mincer/processors/sort'
-require 'mincer/processors/paginate'
-require 'mincer/processors/search'
-require 'mincer/processors/cache_digest'
-require 'mincer/processors/pg_json_dumper'
-::Mincer::Processors.constants.each do |k|
-  klass = ::Mincer::Processors.const_get(k)
-  if klass.is_a?(Class)
-    ::Mincer.processors << klass
-  elsif klass.is_a?(Module)
-    ::Mincer::Base.send(:include, klass)
+  def self.pg_extension_installed?(extension)
+    @installed_extensions ||= {}
+    if @installed_extensions[extension.to_sym].nil?
+      @installed_extensions[extension.to_sym] = ::Mincer.connection.execute("SELECT DISTINCT p.proname FROM pg_proc p WHERE p.proname = '#{extension}'").count > 0
+    end
+    @installed_extensions[extension.to_sym]
   end
+
 end
 
+# Loading helpers
+require 'mincer/processors/helpers'
+
+# Loading processors
+require 'mincer/processors/sorting/processor'
+require 'mincer/processors/pagination/processor'
+require 'mincer/processors/pg_search/search_statement'
+require 'mincer/processors/pg_search/sanitizer'
+require 'mincer/processors/pg_search/search_engines/base'
+require 'mincer/processors/pg_search/search_engines/array'
+require 'mincer/processors/pg_search/search_engines/fulltext'
+require 'mincer/processors/pg_search/search_engines/trigram'
+require 'mincer/processors/pg_search/processor'
+require 'mincer/processors/cache_digest/processor'
+require 'mincer/processors/pg_json_dumper/processor'
 
 # Loading ActionView helpers
 if defined?(ActionView)
@@ -42,19 +58,3 @@ if defined?(ActionView)
     ActionView::Base.send(:include, klass) if klass.is_a?(Module)
   end
 end
-
-
-#if defined?(::Rails)
-#  module Mincer
-#    class Railtie < ::Rails::Railtie
-#      initializer 'mincer.setup_paths' do
-#      end
-#
-#      initializer 'carrierwave.active_record' do
-#        #ActiveSupport.on_load :active_record do
-#        #  require 'carrierwave/orm/activerecord'
-#        #end
-#      end
-#    end
-#  end
-#end

data/mincer.gemspec

@@ -22,10 +22,8 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec', '~> 2.14.1'
   spec.add_development_dependency 'pg'
   spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'kaminari'
   spec.add_development_dependency 'will_paginate'
-  spec.add_development_dependency 'textacular'
 end

data/spec/lib/mincer/action_view/sort_helper_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe Mincer::ActionView::SortHelper do
+  describe 'sort_url_for' do
+    it 'does something' do
+      pending 'Wait for rails integration testing'
+      #helper.url_for
+    end
+  end
+
+end

data/spec/lib/mincer/base_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+
+describe ::Mincer::Base do
+
+  it 'passes all missing methods to relation' do
+    setup_basic_sqlite3_table
+    subject = Class.new(Mincer::Base) do
+      pg_search [{ columns: %w{"active_record_models"."tags" }, engines: [:array] }]
+    end
+    query = subject.new(ActiveRecordModel)
+    expect { query.attribute_names }.not_to raise_exception
+  end
+
+end

data/spec/lib/mincer/config_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+
+describe ::Mincer::Configuration do
+
+
+end