activesearch 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0e0ed236897226e3b2101e34ebed32ca97248f5
-  data.tar.gz: d5b3cee180db19ef832b071c174701a7a039741e
+  metadata.gz: b24050eefed5b21eb1b0d085fa964effdc06c2d8
+  data.tar.gz: 780df5ddeb72407c972869ab11a9a6e015018554
 SHA512:
-  metadata.gz: 41303c2b0d475fc75f348d889e21b57d3c3add61d254fa947b9e137535a8ca65efe63379bd56c135ccba8749b894fd0ebcecd73f9cbe5c6f483c7f5efd9751e8
-  data.tar.gz: 6e638fb66e68d4521299cb10a13f8b740b50888f8f2fbf4c9f800f11758b45a578bc5efba7af1faeb29731acb696dc6d13c8babb9c7e904b40601f186c65e2b9
+  metadata.gz: c1e5576e03f3f0ae6a610ddef63bf3c661e68520ef8fdae55ca4e98c9676f14fb038de8088b416c59b085515ea4568aa5b58500085003bae87ee80c375e66cc7
+  data.tar.gz: f1caac8c129064500ec93a3f13f7854ab83f698ed7e27f8729ae9fc21cf2a2a999a23fd75663304590bb1490ef0e979e2f3a2fff4ff7abd71b4aa884eb7e23c1

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--backtrace

data/Rakefile

@@ -1 +1,19 @@
-require "bundler/gem_tasks"
+#!/usr/bin/env rake
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'rake'
+require 'rspec'
+require 'rspec/core/rake_task'
+require 'rubygems/package_task'
+
+# === Gems install tasks ===
+Bundler::GemHelper.install_tasks
+
+RSpec::Core::RakeTask.new('spec') do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => :spec

data/activesearch.gemspec

@@ -6,8 +6,8 @@ require 'activesearch/version'
 Gem::Specification.new do |gem|
   gem.name          = "activesearch"
   gem.version       = ActiveSearch::VERSION
-  gem.authors       = ["Rodrigo Alvarez"]
-  gem.email         = ["papipo@gmail.com"]
+  gem.authors       = ['Rodrigo Alvarez', 'Didier Lafforgue']
+  gem.email         = ['papipo@gmail.com', 'didier.lafforgue@gmail.com']
   gem.description   = %q{ORM agnostic full text search}
   gem.summary       = %q{ActiveSearch lets you plug in a ruby module in any class that will allow you to do full text searches.}
   gem.homepage      = ""
@@ -16,15 +16,16 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  
+
   gem.add_dependency "activesupport"
   gem.add_dependency "sucker_punch"
   gem.add_dependency "actionpack"
-  
+
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "rspec-mocks"
   gem.add_development_dependency "active_attr"
   gem.add_development_dependency "mongoid", "~> 3"
+  gem.add_development_dependency "database_cleaner", "~> 0.9.1"
   gem.add_development_dependency "tire"
   gem.add_development_dependency "parallel_tests"
   gem.add_development_dependency "httparty"

data/lib/activesearch/algolia/worker.rb

@@ -2,7 +2,7 @@ require "sucker_punch"
 
 class ActiveSearch::Algolia::Worker
   include SuckerPunch::Job
-  
+
   def perform(msg)
     begin
       case msg[:task]
@@ -10,7 +10,7 @@ class ActiveSearch::Algolia::Worker
         ::ActiveSearch::Algolia::Client.new.save(msg[:id], msg[:doc])
       when :deindex
         client = ::ActiveSearch::Algolia::Client.new
-        client.query("", tags: "original_id:#{msg[:id]}")["hits"].each do |hit|
+        client.query("", tags: "original_type:#{msg[:type]},original_id:#{msg[:id]}")["hits"].each do |hit|
           client.delete(hit["objectID"])
         end
       end

data/lib/activesearch/algolia.rb

@@ -4,13 +4,21 @@ require "activesearch/base"
 require "activesearch/proxy"
 
 module ActiveSearch
+
   def self.search(text, conditions = {}, options = {})
+    locale = options[:locale] || I18n.locale
+    conditions[:locale] ||= locale
+
     Proxy.new(text, conditions, options) do |text, conditions|
       Algolia::Client.new.query(text, tags: conditions_to_tags(conditions))["hits"].map! do |hit|
         if hit["_tags"]
           hit["_tags"].each do |tag|
-            k, v = tag.split(':')
-            hit[k] = v
+            # preserve other ":" characters
+            _segments = tag.split(':')
+
+            unless _segments.empty? || _segments[1..-1].empty?
+              hit[_segments.first] = _segments[1..-1].join(':')
+            end
           end
           hit.delete("_tags")
         end
@@ -20,8 +28,9 @@ module ActiveSearch
   end
 
   protected
+
   def self.conditions_to_tags(conditions)
-    conditions.merge(locale: I18n.locale).map { |c| c.join(':') }.join(',')
+    conditions.map { |c| c.join(':') }.join(',')
   end
 
   module Algolia
@@ -33,32 +42,35 @@ module ActiveSearch
 
     protected
     def reindex
-      Worker.new.async.perform(task: :reindex, id: "#{indexable_id}_#{I18n.locale}", doc: to_indexable)
+      Worker.new.async.perform(task: :reindex, id: "#{indexable_id}_#{search_locale}", doc: to_indexable)
     end
 
     def deindex
-      Worker.new.async.perform(task: :deindex, id: indexable_id)
+      Worker.new.async.perform(task: :deindex, id: self.id, type: self.class.to_s)
     end
 
     def to_indexable
-      doc = {}
-      search_fields.each do |field|
-        if send(field)
-          doc[field.to_s] = if send(field).is_a?(Hash) && send(field).has_key?(I18n.locale.to_s)
-            ActiveSearch.strip_tags(send(field)[I18n.locale.to_s])
-          else
-            ActiveSearch.strip_tags(send(field))
+      {}.tap do |doc|
+        _locale = search_locale
+
+        search_fields.each do |field|
+          if content = send(field)
+            doc[field.to_s] = if content.is_a?(Hash) && content.has_key?(_locale)
+              ActiveSearch.strip_tags(content[_locale])
+            else
+              ActiveSearch.strip_tags(content)
+            end
           end
         end
-      end
 
-      (Array(search_options[:store]) - search_fields).each do |field|
-        doc["_tags"] ||= []
-        doc["_tags"] << "#{field}:#{self.send(field)}"
+        (Array(search_options[:store]) - search_fields).each do |field|
+          doc["_tags"] ||= []
+          doc["_tags"] << "#{field}:#{self.send(field)}"
+        end
+        doc["_tags"] << "locale:#{_locale}"
+        doc["_tags"] << "original_type:#{self.class.to_s}"
+        doc["_tags"] << "original_id:#{self.id}"
       end
-      doc["_tags"] << "locale:#{I18n.locale}"
-      doc["_tags"] << "original_id:#{indexable_id}"
-      doc
     end
   end
 end

data/lib/activesearch/base.rb

@@ -7,25 +7,37 @@ module ActiveSearch
       value.gsub(/<\/?[^>]*>/, '')
     when Hash
       value.each_with_object({}) { |(k,v),h| h[k] = strip_tags(v) }
+    when Array
+      value.map { |v| strip_tags(v) }
     else
       value
     end
   end
-  
+
   module Base
     def self.included(parent)
       parent.extend ClassMethods
       parent.class_attribute :search_parameters, instance_reader: false
     end
-    
+
     def search_options
       search_parameters.last.is_a?(Hash) ? search_parameters.last : {}
     end
-    
+
     def search_fields
       search_parameters.last.is_a?(Hash) ? search_parameters[0...-1] : search_parameters
     end
-    
+
+    def search_locale
+      search_locale = search_options[:locale] || I18n.locale.to_s
+
+      if search_locale.respond_to?(:call)
+        search_locale.call
+      else
+        search_locale
+      end
+    end
+
     def search_parameters
       if self.class.search_parameters.is_a?(Symbol)
         self.send(self.class.search_parameters)
@@ -33,14 +45,14 @@ module ActiveSearch
         self.class.search_parameters
       end
     end
-    
+
     module ClassMethods
       def search_by(params, conditions = {})
         after_save    :reindex, conditions
         after_destroy :deindex, conditions
         self.search_parameters = params
       end
-      
+
     end
   end
 end

data/lib/activesearch/mongoid/index.rb

@@ -0,0 +1,115 @@
+module ActiveSearch
+  module Mongoid
+    class Index
+
+      include ::Mongoid::Document
+
+      ## fields ##
+      field :original_type, type: String
+      field :original_id,   type: Moped::BSON::ObjectId
+      field :language,      type: String
+      field :locale,        type: String
+      field :content,       type: Array
+      field :stored,        type: Hash, default: {}
+      alias_method :to_hash, :stored
+
+      ## indexes ##
+      index({ content: 'text', locale: 1 })
+      index({ original_type: 1, original_id: 1, locale: 1 }, unique: true)
+
+      ## methods ##
+
+      def store_language(original)
+        self.locale   = original.search_locale
+        self.language = self.class.locale_to_language(self.locale)
+      end
+
+      def store_fields(original)
+        self.stored = {}
+
+        fields = (original.search_fields + (original.search_options[:store] || [])).uniq
+
+        fields.each do |f|
+          if original.send(f).present?
+            self.stored[f] = ActiveSearch.strip_tags(original.send(f))
+          end
+        end
+      end
+
+      def refresh_content(original)
+        self.content = original.to_indexable.values.flatten
+      end
+
+      ## class methods ##
+
+      def self.search(query, conditions = {})
+        language = self.locale_to_language(conditions[:locale] || I18n.locale)
+
+        filter = {}
+        conditions.each do |key, value|
+          if key == :locale
+            filter['locale'] = value.to_s
+          else
+            filter["stored.#{key}"] = value
+          end
+        end
+
+        session = ::Mongoid.session('default')
+        results = session.command({
+          'text'      => collection.name,
+          'search'    => query,
+          'language'  => language,
+          'filter'    => filter
+        })
+        if results.has_key?('results')
+          results['results'].map do |result|
+            result['obj']['stored'].merge(result['obj'].slice('locale', 'original_type', 'original_id'))
+          end
+        else
+          []
+        end
+      end
+
+      def self.deindex(original)
+        # delete the records in all the locales
+        self.where(original_type: original.class.to_s, original_id: original.id).destroy
+      end
+
+      def self.reindex(original, fields, options)
+        # re-index only in the current locale (unless another locale has been specified)
+        locale = original.search_locale
+
+        # find the exact index scoped by the locale or build a new one
+        doc = find_or_initialize_by(original_type: original.class.to_s, original_id: original.id, locale: locale)
+
+        doc.store_language(original)
+        doc.store_fields(original) #, fields, options)
+        doc.refresh_content(original)
+
+        # save it (create or update it)
+        doc.save
+      end
+
+      def self.locale_to_language(locale)
+        {
+          dk: 'danish',
+          nl: 'dutch',
+          en: 'english',
+          fi: 'finnish',
+          fr: 'french',
+          de: 'german',
+          hu: 'hungarian',
+          it: 'italian',
+          nb: 'norwegian',
+          br: 'portuguese',
+          pt: 'portuguese',
+          ro: 'romanian',
+          ru: 'russian',
+          es: 'spanish',
+          se: 'swedish',
+          tr: 'turkish'
+        }[locale.to_sym] || 'english'
+      end
+    end
+  end
+end

data/lib/activesearch/mongoid.rb

@@ -1,15 +1,15 @@
 require 'activesearch/base'
 require 'activesearch/proxy'
-require 'activesearch/mongoid/model'
+require 'activesearch/mongoid/index'
 
 module ActiveSearch
 
   def self.search(text, conditions = {}, options = {})
+    locale = options[:locale] || I18n.locale
+    conditions[:locale] ||= locale
+
     Proxy.new(text, conditions, options) do |text, conditions|
-      text = text.downcase.split(/\s+/)
-      conditions.keys.each { |k| conditions["_stored.#{k}"] = conditions.delete(k) }
-      conditions.merge!(:_keywords.in => text + text.map { |word| "#{I18n.locale}:#{word}"})
-      Mongoid::Model.where(conditions)
+      ActiveSearch::Mongoid::Index.search(text, conditions)
     end
   end
 
@@ -20,13 +20,30 @@ module ActiveSearch
       end
     end
 
+    def to_indexable
+      {}.tap do |doc|
+        _locale = search_locale
+
+        search_fields.each do |field|
+          if content = send(field)
+            doc[field.to_s] = if content.is_a?(Hash) && content.has_key?(_locale)
+              ActiveSearch.strip_tags(content[_locale])
+            else
+              ActiveSearch.strip_tags(content)
+            end
+          end
+        end
+      end
+    end
+
     protected
+
     def reindex
-      ActiveSearch::Mongoid::Model.reindex(self, self.search_fields, self.search_options)
+      ActiveSearch::Mongoid::Index.reindex(self, self.search_fields, self.search_options)
     end
 
     def deindex
-      ActiveSearch::Mongoid::Model.deindex(self)
+      ActiveSearch::Mongoid::Index.deindex(self)
     end
   end
 end

data/lib/activesearch/result.rb

@@ -1,5 +1,6 @@
 require 'action_view'
 require 'active_support/core_ext'
+require 'active_support/core_ext/hash/slice'
 
 module ActiveSearch
   class Result < Hash
@@ -9,16 +10,22 @@ module ActiveSearch
     include ActionView::Helpers::TextHelper
 
     def initialize(result, text, options = {})
+      locale = (options[:locale] || I18n.locale).to_s
+
       @text = text
       result.to_hash.each do |k,v|
-        unless v.nil? || k.to_s.start_with?('_')
-          self[k.to_s] = v.respond_to?(:has_key?) && v.has_key?(I18n.locale.to_s) ? v[I18n.locale.to_s] : v
+        unless v.nil?
+          self[k.to_s] = v.respond_to?(:has_key?) && v.has_key?(locale) ? v[locale] : v
         end
       end
 
       self.build_highlighted_fields(options[:radius])
     end
 
+    def slice(*keys)
+      ::Hash.new.update(self).slice(*keys)
+    end
+
     protected
 
     def build_highlighted_fields(radius = nil)

data/lib/activesearch/version.rb

@@ -1,3 +1,3 @@
 module ActiveSearch
-  VERSION = "0.2.0"
+  VERSION = '0.3.0'
 end

data/spec/engines/algolia_spec.rb

@@ -0,0 +1,32 @@
+#encoding: utf-8
+require 'spec_helper'
+
+describe 'ActiveSearch::Algolia' do
+
+  before(:all) do
+    SetupEngine.setup(:algolia)
+
+    # can not use the rspec let method
+    @model          = AlgoliaModel
+    @another_model  = AnotherAlgoliaModel
+  end
+
+  include_examples 'an engine'
+
+  context 'retry on errors' do
+
+    before do
+      times_called = 0
+      @instance = AlgoliaModel.new(title: 'Example')
+      ActiveSearch::Algolia::Client.should_receive(:put).exactly(3).times.and_return do
+        times_called += 1
+        raise Errno::ECONNRESET if times_called <= 2
+      end
+    end
+
+    subject { -> { @instance.save } }
+
+    it { should_not raise_error }
+  end
+
+end

data/spec/{base_spec.rb → engines/base_spec.rb}

@@ -1,64 +1,64 @@
-require 'activesearch/base'
+require 'spec_helper'
 
 describe ActiveSearch::Base do
   before do
     @klass = Class.new do
       include ActiveSearch::Base
-      
+
       def self.after_save(*args); end
       def self.after_destroy(*args); end
-      
+
     end
   end
-  
+
   context "search_by" do
     let(:call_search_by) do
       @klass.class_eval do
         search_by [:field], if: :something_happens, unless: :its_friday
       end
     end
-    
+
     it "should rely on after_save and after_destroy callbacks passing conditions" do
       @klass.should_receive(:after_save).with(:reindex, if: :something_happens, unless: :its_friday)
       @klass.should_receive(:after_destroy).with(:deindex, if: :something_happens, unless: :its_friday)
       call_search_by
     end
-    
+
     it "should store the parameters in search_parameters" do
       call_search_by
       @klass.send(:search_parameters).should == [:field]
     end
   end
-  
+
   context "utility methods with options" do
     before do
       @klass.stub(:search_parameters).and_return([:field, store: [:another_field]])
     end
-    
+
     it "search_options should return the hash at the end of the parameters" do
       @klass.new.search_options.should == {store: [:another_field]}
     end
-    
+
     it "search_fields should return all parameters except the options" do
       @klass.new.search_fields.should == [:field]
     end
   end
-  
+
   context "search_by with virtual parameters" do
     before do
       @klass.class_eval do
         def options_for_search
           [@field]
         end
-        
+
         def initialize(field)
           @field = field
         end
-        
+
         search_by :options_for_search
       end
     end
-    
+
     it "should work" do
       @klass.new(:first).send(:search_parameters).should == [:first]
       @klass.new(:second).send(:search_parameters).should == [:second]

data/spec/engines/mongoid_spec.rb

@@ -0,0 +1,56 @@
+#encoding: utf-8
+require 'spec_helper'
+
+describe 'ActiveSearch::Mongoid' do
+
+  before(:all) do
+    SetupEngine.setup(:mongoid)
+
+    # can not use the rspec let method
+    @model          = MongoidModel
+    @another_model  = AnotherMongoidModel
+  end
+
+  include_examples 'an engine'
+
+  describe 'localized content' do
+
+    before(:all) do
+      @localized = LocalizedMongoidModel.create!(title: "<strong>English</strong> English")
+      I18n.with_locale(:es) do
+        @localized.title = "Español Español"
+        @localized.save!
+      end
+    end
+
+    it "should be able to find by different locales" do
+      ActiveSearch.search("english").first["title"].should == "English English"
+      I18n.with_locale(:es) do
+        ActiveSearch.search("español").first["title"].should == "Español Español"
+      end
+    end
+
+    it "finds by a different locale" do
+      ActiveSearch.search("español", {}, { locale: 'es'}).first["title"].should == "Español Español"
+    end
+
+    it "should store content with tags stripped" do
+      index = ActiveSearch::Mongoid::Index.where(original_type: "LocalizedMongoidModel", original_id: @localized.id, locale: 'en')
+      index.first.content.should == ["English English"]
+    end
+
+    it "handles empty translations" do
+      lambda { LocalizedMongoidModel.create!(title: nil, not_localized: "example") }.should_not raise_error
+    end
+
+    it "handles empty fields" do
+      lambda { LocalizedMongoidModel.create!(title: "Example", not_localized: nil) }.should_not raise_error
+    end
+
+    it "handles nil values in arrays" do
+      lambda { LocalizedMongoidModel.create!(title: "Example", not_localized: "example", array: [nil]) }.should_not raise_error
+    end
+
+  end
+
+end

data/spec/models/algolia.rb

@@ -1,8 +1,6 @@
-require 'activesearch/algolia'
-
 class AlgoliaModel < ActiveMimic
   include ActiveSearch::Algolia
-  
+
   attribute :title
   attribute :text
   attribute :junk
@@ -10,19 +8,19 @@ class AlgoliaModel < ActiveMimic
   attribute :scope_id, type: Integer
   attribute :tags, type: Array
   localized_attribute :color
-  
+
   search_by [:title, :text, :tags, :color, store: [:title, :junk, :scope_id]], if: lambda { !self.special }
 
 end
 
 class AnotherAlgoliaModel < ActiveMimic
   include ActiveSearch::Algolia
-  
+
   attribute :title, type: String
   attribute :scope_id, type: Integer
   localized_attribute :color
   search_by [:title, store: [:title, :virtual, :scope_id, :color]]
-  
+
   def virtual
     "virtual"
   end

data/spec/models/elastic_search.rb

@@ -1,11 +1,9 @@
-require 'activesearch/elastic_search'
-
 module ElasticSearchRefresh
-  
+
   def save
     super.tap { Tire.index('_all') { refresh }}
   end
-  
+
   def destroy
     super.tap { Tire.index('_all') { refresh }}
   end
@@ -14,13 +12,12 @@ end
 class ElasticSearchModel < ActiveMimic
   include ActiveSearch::ElasticSearch
   include ElasticSearchRefresh
-  
+
   attribute :title
-  attribute :text
   attribute :junk
   attribute :special, default: false
   attribute :tags, type: Array
-  
+
   search_by [:title, :text, :tags, store: [:title, :junk]], if: lambda { !self.special }
 
 end
@@ -28,10 +25,10 @@ end
 class AnotherElasticSearchModel < ActiveMimic
   include ActiveSearch::ElasticSearch
   include ElasticSearchRefresh
-  
+
   attribute :title, type: String
   search_by [:title, store: [:title, :virtual]]
-  
+
   def virtual
     "virtual"
   end