redis-search 0.9.6 → 0.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f06b8b05df240f527992050fa3174ae7726cb0e
-  data.tar.gz: 1f37d2e08628b4a5fe6a9a5416a88f5c6b1b989d
+  metadata.gz: 715f373324e8d649a86d38d05ce25e044b4ebacc
+  data.tar.gz: bc32f91272a550a62f697366df1c807d2b7c58c2
 SHA512:
-  metadata.gz: ec74e499cf300aeab20a0851c2228f0bacbbae75683d4517f44894bb4c331f83c753ae98c112b342762e59457b9adf36b645728dcf495b890d516a52351871fe
-  data.tar.gz: c6ea104e95e3d83ba984a18522925dc881eb37fb021750c5b1476b14c8ae90eab70d07b14fe88c56a3bd3e298f3923a8fb83236b52c0a5033370e382be9bf63a
+  metadata.gz: dc3272074ff70fd82998bd46d7b566a6072ceb7c3820411e8169386f395cecf98896100934edae3e15e1bf50406a7e3556de1c15f0f32e402deb01d548f882ad
+  data.tar.gz: c115f7733556b42db81f707d0a857697e9881cf78b5e7dabbd3b2d67dd3d72d953e84a4b04a98af23190aae3546287abaf9c8902f36c93b0a19ec82d0a5385fe

data/README.md

@@ -107,7 +107,7 @@ And there is an  [Example App](https://github.com/huacnlee/redis-search-example)
       field :followers_count
 
       redis_search_index(:title_field => :name,
-                     :alias_field => :alias_names,
+                         :alias_field => :alias_names,
                          :prefix_index_enable => true,
                          :score_field => :followers_count,
                          :ext_fields => [:email,:tagline])
@@ -130,10 +130,32 @@ And there is an  [Example App](https://github.com/huacnlee/redis-search-example)
 
 ## Index data to Redis
 
-If you are first install it in you old project, or your Redis cache lose, you can use this command to rebuild indices.
+### Specify Model
+
+Redis-Search index data to Redis from your model (pass name as CLASS environment variable).
+
+```bash
+$ rake redis_search:index:model CLASS='MyModel'
+```
+
+Customize the batch size:
 
 ```bash
-$ rake redis_search:index
+$ rake redis_search:index:model CLASS='MyModel' BATCH=100
+```
+
+### All Models
+
+Redis-Search all index data to Redis from `app/models` (or use DIR environment variabl).
+
+```bash
+$ rake redis_search:index DIR=app/models
+```
+
+Customize the batch size:
+
+```bash
+$ rake redis_search:index DIR=app/models BATCH=100
 ```
 
 ## Documentation
@@ -148,7 +170,6 @@ There is my performance test result.
 
 * [https://gist.github.com/1150933](https://gist.github.com/1150933)
 
-
 ## License
 
 * MIT

data/lib/redis/search/base.rb

@@ -1,134 +1,168 @@
 # coding: utf-8
-
 class Redis
   module Search
     autoload :PinYin, 'ruby-pinyin'
 
     extend ::ActiveSupport::Concern
 
+    included do
+      cattr_reader :redis_search_options
+
+      before_destroy :redis_search_index_before_destroy
+      after_update :redis_search_index_after_update
+      after_save :redis_search_index_after_save
+    end
+
+    def redis_search_fields_to_hash(ext_fields)
+      exts = {}
+      ext_fields.each do |f|
+        exts[f] = instance_eval(f.to_s)
+      end
+      exts
+    end
+
+    def redis_search_alias_value(field)
+      return [] if field.blank? || field == "_was".freeze
+      val = (instance_eval("self.#{field}") || "".freeze).clone
+      return [] if !val.class.in?([String,Array])
+      if val.is_a?(String)
+        val = val.to_s.split(",")
+      end
+      val
+    end
+
+    # Rebuild search index with create
+    def redis_search_index_create
+      s = Search::Index.new(title: self.send(self.redis_search_options[:title_field]),
+                            aliases: self.redis_search_alias_value(self.redis_search_options[:alias_field]),
+                            id: self.id,
+                            exts: self.redis_search_fields_to_hash(self.redis_search_options[:ext_fields]),
+                            type: self.redis_search_options[:class_name] || self.class.name,
+                            condition_fields: self.redis_search_options[:condition_fields],
+                            score: self.send(self.redis_search_options[:score_field]).to_i,
+                            prefix_index_enable: self.redis_search_options[:prefix_index_enable])
+      s.save
+      # release s
+      s = nil
+      true
+    end
+
+    def redis_search_index_delete(titles)
+      titles.uniq!
+      titles.each do |title|
+        next if title.blank?
+        Search::Index.remove(id: self.id, title: title, type: self.class.name)
+      end
+      true
+    end
+
+    def redis_search_index_before_destroy
+      titles = []
+      titles = redis_search_alias_value(self.redis_search_options[:alias_field])
+      titles << self.send(self.redis_search_options[:title_field])
+
+      redis_search_index_delete(titles)
+      true
+    end
+
+    def redis_search_index_need_reindex
+      index_fields_changed = false
+      self.redis_search_options[:ext_fields].each do |f|
+        next if f.to_s == "id".freeze
+        field_method = "#{f}_changed?"
+        if self.methods.index(field_method.to_sym) == nil
+          Redis::Search.warn("#{self.class.name} model reindex on update need #{field_method} method.")
+          next
+        end
+
+        index_fields_changed = true if instance_eval(field_method)
+      end
+
+      begin
+        if self.send("#{self.redis_search_options[:title_field]}_changed?")
+          index_fields_changed = true
+        end
+
+        if self.send(self.redis_search_options[:alias_field]) || self.send("#{self.redis_search_options[:title_field]}_changed?")
+          index_fields_changed = true
+        end
+      rescue
+      end
+
+      return index_fields_changed
+    end
+
+    def redis_search_index_after_update
+      if self.redis_search_index_need_reindex
+        titles = []
+        titles = redis_search_alias_value("#{self.redis_search_options[:alias_field]}_was")
+        titles << self.send("#{self.redis_search_options[:title_field]}_was")
+        redis_search_index_delete(titles)
+      end
+
+      true
+    end
+
+    def redis_search_index_after_save
+      if self.redis_search_index_need_reindex || self.new_record?
+        self.redis_search_index_create
+      end
+      true
+    end
+
     module ClassMethods
       # Config redis-search index for Model
       # == Params:
       #   title_field   Query field for Search
-      #   alias_field   Alias field for search, can accept multi field (String or Array type), it type is String, redis-search will split by comma
+      #   alias_field   Alias field for search, can accept multi field (String or Array type) it type is String, redis-search will split by comma
       #   prefix_index_enable   Is use prefix index search
       #   ext_fields    What kind fields do you need inlucde to search indexes
       #   score_field   Give a score for search sort, need Integer value, default is `created_at`
-      def redis_search_index(options = {})
-        title_field         = options[:title_field] || :title
-        alias_field         = options[:alias_field] || nil
-        prefix_index_enable = options[:prefix_index_enable] || false
-        ext_fields          = options[:ext_fields] || []
-        score_field         = options[:score_field] || :created_at
-        condition_fields    = options[:condition_fields] || []
+      def redis_search_index(opts = {})
+        opts[:title_field] ||= :title
+        opts[:alias_field] ||= nil
+        opts[:prefix_index_enable] ||= false
+        opts[:ext_fields] ||= []
+        opts[:score_field] ||= :created_at
+        opts[:condition_fields] ||= []
+        opts[:class_name] ||= nil
 
         # Add score field to ext_fields
-        ext_fields |= [score_field]
-        # Add condition fields to ext_fields
-        ext_fields |= condition_fields
+        opts[:ext_fields] += [opts[:score_field]]
 
-        if RUBY_VERSION.start_with?('1.8')
-          condition_fields =  "[#{condition_fields.collect { |c| "'#{c}'" }.join(',')}]"
-        end
+        # Add condition fields to ext_fields
+        opts[:ext_fields] += opts[:condition_fields] if opts[:condition_fields].is_a?(Array)
 
         # store Model name to indexed_models for Rake tasks
         Search.indexed_models = [] if Search.indexed_models == nil
         Search.indexed_models << self
-        # bind instance methods and callback events
-        class_eval %(
-          def redis_search_fields_to_hash(ext_fields)
-            exts = {}
-            ext_fields.each do |f|
-              exts[f] = instance_eval(f.to_s)
-            end
-            exts
-          end
-
-          def redis_search_alias_value(field)
-            return [] if field.blank? || field == "_was"
-            val = (instance_eval("self.\#{field}") || "").clone
-            return [] if !val.class.in?([String,Array])
-            if val.is_a?(String)
-              val = val.to_s.split(",")
-            end
-            val
-          end
-
-          def redis_search_index_create
-            s = Search::Index.new(:title => self.#{title_field},
-                                  :aliases => self.redis_search_alias_value(#{alias_field.inspect}),
-                                  :id => self.id,
-                                  :exts => self.redis_search_fields_to_hash(#{ext_fields.inspect}),
-                                  :type => self.class.to_s,
-                                  :condition_fields => #{condition_fields},
-                                  :score => self.#{score_field}.to_i,
-                                  :prefix_index_enable => #{prefix_index_enable})
-            s.save
-            # release s
-            s = nil
-            true
-          end
-
-          def redis_search_index_delete(titles)
-            titles.uniq.each do |title|
-              next if title.blank?
-              Search::Index.remove(:id => self.id, :title => title, :type => self.class.to_s)
-            end
-            true
-          end
-
-          before_destroy do
-            titles = []
-            titles = redis_search_alias_value("#{alias_field}")
-            titles << self.#{title_field}
 
-            redis_search_index_delete(titles)
-            true
-          end
+        class_variable_set("@@redis_search_options".freeze, opts)
+      end
 
-          def redis_search_index_need_reindex
-            index_fields_changed = false
-            #{ext_fields.inspect}.each do |f|
-              next if f.to_s == "id"
-              field_method = f.to_s + "_changed?"
-              if self.methods.index(field_method.to_sym) == nil
-                Search.warn("#{self.class.name} model reindex on update need "+field_method+" method.")
-                next
-              end
-              if instance_eval(field_method)
-                index_fields_changed = true
-              end
+      def redis_search_index_batch_create(batch_size = 1000, progressbar = false)
+        count = 0
+        if self.ancestors.collect { |klass| klass.to_s }.include?("ActiveRecord::Base".freeze)
+          find_in_batches(:batch_size => batch_size) do |items|
+            items.each do |item|
+              item.redis_search_index_create
+              count += 1
+              print "." if progressbar
             end
-            begin
-              if self.#{title_field}_changed?
-                index_fields_changed = true
-              end
-              if self.#{alias_field || title_field}_changed?
-                index_fields_changed = true
-              end
-            rescue
-            end
-            return index_fields_changed
           end
-
-          after_update do
-            if self.redis_search_index_need_reindex
-              titles = []
-              titles = redis_search_alias_value("#{alias_field}_was")
-              titles << self.#{title_field}_was
-              redis_search_index_delete(titles)
+        elsif self.included_modules.collect { |m| m.to_s }.include?("Mongoid::Document".freeze)
+          all.each_slice(batch_size) do |items|
+            items.each do |item|
+              item.redis_search_index_create
+              count += 1
+              print "." if progressbar
             end
-            true
           end
+        else
+          puts "skiped, not support this ORM in current."
+        end
 
-          after_save :redis_search_index_update
-          def redis_search_index_update
-            if self.redis_search_index_need_reindex || self.new_record?
-              self.redis_search_index_create
-            end
-            true
-          end
-        )
+        count
       end
     end
   end

data/lib/redis/search/finder.rb

@@ -19,7 +19,7 @@ class Redis
       # * Redis::Search.complete("Tag","red") => ["Redis", "Redmine"]
       # * Redis::Search.complete("Tag","redi") => ["Redis"]
       def complete(type, w, options = {})
-        limit = options[:limit] || 10
+        limit      = options[:limit] || 10
         conditions = options[:conditions] || []
         return [] if (w.blank? && conditions.blank?) || type.blank?
 
@@ -89,9 +89,9 @@ class Redis
         end
 
         ids = self.config.redis.sort(temp_store_key,
-                                    :limit => [0,limit],
-                                    :by => self.mk_score_key(type,"*"),
-                                    :order => "desc")
+                                     limit: [0,limit],
+                                     by: self.mk_score_key(type,"*"),
+                                     order: "desc")
         return [] if ids.blank?
         self.hmget(type,ids)
       end
@@ -105,10 +105,9 @@ class Redis
       # h3. usage:
       # * Redis::Search.query("Tag","Ruby vs Python")
       def query(type, text, options = {})
-        tm = Time.now
-        result = []
-
-        limit = options[:limit] || 10
+        tm         = Time.now
+        result     = []
+        limit      = options[:limit] || 10
         sort_field = options[:sort_field] || "id"
         conditions = options[:conditions] || []
 
@@ -164,10 +163,10 @@ class Redis
 
         # 根据需要的数量取出 ids
         ids = self.config.redis.sort(temp_store_key,
-                                    :limit => [0,limit],
-                                    :by => self.mk_score_key(type,"*"),
-                                    :order => "desc")
-        result = self.hmget(type,ids, :sort_field => sort_field)
+                                     limit: [0,limit],
+                                     by: self.mk_score_key(type,"*"),
+                                     order: "desc")
+        result = self.hmget(type, ids, sort_field: sort_field)
         self.info("{#{type} : \"#{text}\"} | Time spend: #{Time.now - tm}s")
         result
       end

data/lib/redis/search/index.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 class Redis
   module Search
     class Index
@@ -70,7 +71,7 @@ class Redis
         return if @title.blank?
 
         self.redis.pipelined do
-          data = {:title => @title, :id => @id, :type => @type}
+          data = {title: @title, id: @id, type: @type}
           self.exts.each do |f|
             data[f[0]] = f[1]
           end

data/lib/redis/search/railtie.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 class Redis
   module Search
     class Railtie < ::Rails::Railtie

data/lib/redis/search/tasks.rb

@@ -1,42 +1,73 @@
 # coding: utf-8
 require "redis-search"
 namespace :redis_search do
-  desc "Redis-Search index data to Redis"
-  task :index => :environment do
-    tm = Time.now
-    count = 0
-    puts "redis-search index".upcase.rjust(120)
-    puts "-"*120
-    puts "Now indexing search to Redis...".rjust(120)
-    puts ""
-    Redis::Search.indexed_models.each do |klass|
-      print "[#{klass.to_s}]"
-      if klass.superclass.to_s == "ActiveRecord::Base"
-        klass.find_in_batches(:batch_size => 1000) do |items|
-          items.each do |item|
-            item.redis_search_index_create
-      			item = nil
-      			count += 1
-            print "."
-          end
-        end
-      elsif klass.included_modules.collect { |m| m.to_s }.include?("Mongoid::Document")
-        klass.all.each_slice(1000) do |items|
-          items.each do |item|
-            item.redis_search_index_create
-      			item = nil
-      			count += 1
-            print "."
-          end
-        end
-      else
-        puts "skiped, not support this ORM in current."
+  task index: 'index:all'
+
+  namespace :index do
+    index_model_desc = <<-DESC.gsub(/      /, '')
+      Redis-Search index data to Redis from your model (pass name as CLASS environment variable).
+
+        $ rake environment redis_search:index:model CLASS='MyModel'
+
+      Customize the batch size:
+
+        $ rake environment redis_search:index:model CLASS='Article' BATCH=100
+    DESC
+
+    index_all_desc = <<-DESC.gsub(/      /, '')
+      Redis-Search all index data to Redis from `app/models` (or use DIR environment variabl).
+
+        $ rake environment redis_search:index:all DIR=app/models
+
+      Customize the batch size:
+
+        $ rake environment redis_search:index:all DIR=app/models BATCH=100
+    DESC
+
+    desc index_model_desc
+    task model: :environment do
+      if ENV['CLASS'].to_s == ''
+        puts '='*90, 'USAGE', '='*90, index_model_desc, ""
+        exit(1)
       end
+
+      klass = eval(ENV['CLASS'].to_s)
+      batch = ENV['BATCH'].to_i > 0 ? ENV['BATCH'].to_i : 1000
+      tm    = Time.now
+      puts "Redis-Search index data to Redis from [#{klass.to_s}]"
+      count = klass.redis_search_index_batch_create(batch, true)
       puts ""
+      puts "Indexed #{count} rows  |  Time spend: #{(Time.now - tm)}s"
+      puts "Rebuild Index done."
+    end
+
+    desc index_all_desc
+    task all: :environment do
+      tm    = Time.now
+      count = 0
+      dir   = ENV['DIR'].to_s != '' ? ENV['DIR'] : 'app/models'
+      batch = ENV['BATCH'].to_i > 0 ? ENV['BATCH'].to_i : 1000
+
+      Dir.glob(File.join("#{dir}/**/*.rb")).each do |path|
+        model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1]
+
+        next if model_filename.match(/^concerns\//i) # Skip concerns/ folder
+
+        begin
+          klass = model_filename.camelize.constantize
+        rescue NameError
+          require(path) ? retry : raise(RuntimeError, "Cannot load class '#{klass}'")
+        end
+      end
+
+      puts "Redis-Search index data to Redis from [#{dir}]"
+      Redis::Search.indexed_models.each do |klass|
+        puts "[#{klass.to_s}]"
+        count += klass.redis_search_index_batch_create(batch, true)
+        puts ""
+      end
+      puts "Indexed #{count} rows  |  Time spend: #{(Time.now - tm)}s"
+      puts "Rebuild Index done."
     end
-    puts ""
-    puts "-"*120
-    puts "Indexed #{count} rows  |  Time spend: #{(Time.now - tm)}s".rjust(120)
-    puts "Rebuild Index done.".rjust(120)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-search
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.9.7
 platform: ruby
 authors:
 - Jason Lee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-01 00:00:00.000000000 Z
+date: 2014-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-pinyin