redis-autosuggest 0.1.0 → 0.2.0

data/README.md

@@ -5,16 +5,6 @@ Provides autocompletions through Redis, with the ability to rank
 
 ## Installation
 
-Add this line to your application's Gemfile:
-
-    gem 'redis-autosuggest'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
     $ gem install redis-autosuggest
 
 ## Usage
@@ -42,9 +32,9 @@ Autocompletions will be ordered their score value (descending).
 Some other usage examples:
 ```ruby
 # Add items with initial scores 
-Redis::Autosuggest.suggest("North By Northwest", 9, Northern Exposure, 3)
+Redis::Autosuggest.add_with_score("North By Northwest", 9, Northern Exposure, 3)
 # Increment an item's score
-Redis::Autosuggest.suggest("North By Northwest", 1)
+Redis::Autosuggest.increment("North By Northwest", 1)
 ```
 
 ## Rails support
@@ -61,19 +51,14 @@ end
 
 For first time usage, seed the Redis db with the autosuggest sources:
 ```ruby
-Redis::Autosuggest.init_rails_sources
+rake autosuggest:init
 ```
 
 You can optionally specify a numeric field to be used as the initial score for an item
-when it is added:
+when it is added and a limit of how many items maximum to keep:
 ```ruby
-  autosuggest     :movie_title, :rank_by => imdb_rating
+  autosuggest :movie_title, :rank_by => imdb_rating, limit => 10000
 ```
 
-## Contributing
-
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+## Front-end portion
+Jquery plugin for dropdown autocompletions for a from can be found [here](https://github.com/aphan/jquery-rtsuggest)

data/lib/redis-autosuggest.rb

@@ -3,9 +3,9 @@ require 'redis-namespace'
 require 'redis/autosuggest'
 require 'redis/autosuggest/config'
 require 'redis/autosuggest/file'
-require 'redis/autosuggest/init'
 require 'redis/autosuggest/version'
 
 if defined?(Rails)
-  require 'redis/autosuggest/rails'
+  require 'redis/autosuggest/rails/sources'
+  require 'redis/autosuggest/rails/railtie'
 end

data/lib/redis/autosuggest.rb

@@ -4,48 +4,54 @@ class Redis
     class << self
 
       # Add item(s) to the pool of items to autosuggest from.  Each item's initial
-      # rank is 0
+      # rank is 0. Returns true if all items added were new, false otherwise.
       def add(*items)
-        item_pool = @db.hgetall(@items).values
-        items.each do |i|
-          next if item_pool.include?(i.downcase)
-          add_item(i.downcase)
+        all_new_items = true
+        items.each do |item|
+          item = item.downcase
+          item_exists?(item) ? all_new_items = false : add_item(item)
         end
+        all_new_items
       end
 
-      # Add item(s) along with their scores.
+      # Add item(s) along with their initial scores.
+      # Returns true if all items added were new, false otherwise.
       # add_with_score("item1", 4, "item2", 1, "item3", 0)
       def add_with_score(*fields)
-        item_pool = @db.hgetall(@items).values
+        all_new_items = true 
         fields.each_slice(2) do |f|
-          next if item_pool.include?(f[0].downcase)
-          add_item(f[0].downcase, f[1])
+          f[0] = normalize(f[0])
+          item_exists?(f[0]) ? all_new_items = false : add_item(*f)
         end
+        all_new_items
       end
 
-      # Remove an item from the pool of items to autosuggest from
+      # Remove an item from the pool of items to autosuggest from.
+      # Returns true if an item was indeed removed, false otherwise.
       def remove(item)
         item = item.downcase
         id = get_id(item)
-        return if id.nil?
+        return false if id.nil?
         @db.hdel(@items, id)
+        @db.hdel(@itemids, item)
         remove_substrings(item, id)
         @redis.zrem(@leaderboard, id) if @use_leaderboard
+        return true
       end
 
-      # Increment the score (by 1 by default) of an item.  Pass in a negative value
-      # to decrement the score
-      def increment(item, inc=1)
-        item = item.downcase
+      # Increment the score (by 1 by default) of an item.  
+      # Pass in a negative value to decrement the score.
+      def increment(item, incr=1)
+        item = normalize(item)
         id = get_id(item)
-        each_substring(item) { |sub| @substrings.zincrby(sub, inc, id) }
-        @db.zincrby(@leaderboard, inc, id) if @use_leaderboard
+        each_substring(item) { |sub| @substrings.zincrby(sub, incr, id) }
+        @db.zincrby(@leaderboard, incr, id) if @use_leaderboard
       end
 
       # Suggest items from the database that most closely match the queried string.
-      # Returns an array of suggestion items (an empty array if nothing found)
+      # Returns an array of suggestion items (an empty array if nothing found).
       def suggest(str, results=@max_results)
-        suggestion_ids = @substrings.zrevrange(str.downcase, 0, results - 1)
+        suggestion_ids = @substrings.zrevrange(normalize(str), 0, results - 1)
         suggestion_ids.empty? ? [] : @db.hmget(@items, suggestion_ids)
       end
 
@@ -57,15 +63,32 @@ class Redis
 
       # Get the score of an item
       def get_score(item)
-        @substrings.zscore(item.downcase, get_id(item.downcase))
+        item = normalize(item)
+        @substrings.zscore(item, get_id(item))
+      end
+
+      # Returns whether or not an item is already stored in the db
+      def item_exists?(item)
+        return !get_id(normalize(item)).nil?
+      end
+
+      # Get the id associated with an item in the db
+      def get_id(item)
+        return @db.hmget(@itemids, normalize(item)).first
       end
 
       private
+
+      def normalize(item)
+        return item.downcase.strip
+      end
+
       def add_item(item, score=0)
-        id = self.db.hlen(self.items)
-        self.db.hset(self.items, id, item)
+        id = @db.hlen(@items)
+        @db.hset(@items, id, item)
+        @db.hset(@itemids, item, id)
         add_substrings(item, score, id)
-        self.db.zadd(self.leaderboard, score, id) if self.use_leaderboard
+        @db.zadd(@leaderboard, score, id) if @use_leaderboard
       end
 
       # Yield each substring of a complete string 
@@ -75,19 +98,42 @@ class Redis
 
       # Add all substrings of a string to redis
       def add_substrings(str, score, id)
-        each_substring(str) { |sub| @substrings.zadd(sub, score, id) }
+        each_substring(str) do |sub| 
+          if @max_per_substring == Float::INFINITY
+            add_substring(sub, score, id)
+          else
+            add_substring_limit(sub, score, id)
+          end
+        end
+      end
+      
+      # Add the id of an item to a substring
+      def add_substring(sub, score, id)
+        @substrings.zadd(sub, score, id)
+      end
+      
+      # Add the id of an item to a substring only when the number of items that
+      # substring stores is less then the config value of "max_per_substring".
+      # If the substring set is already full, check to see if the item with the
+      # lowest score in the substring set has a lower score than the item being added.
+      # If yes, remove that item and add this item to the substring set.
+      def add_substring_limit(sub, score, id)
+        count = @substrings.zcount(sub, "-inf", "inf")
+        if count < @max_per_substring
+          add_substring(sub, score, id)
+        else
+          lowest_item = @substrings.zrevrange(sub, -1, -1, { withscores: true }).last
+          if score > lowest_item[1]
+            @substrings.zrem(sub, lowest_item[0])
+            add_substring(sub, score, id)
+          end
+        end
       end
 
       # Remove all substrings of a string from the db
       def remove_substrings(str, id)
         each_substring(str) { |sub| @substrings.zrem(sub, id) }
       end
-
-      # Get the id associated with an item in the db 
-      def get_id(item)
-        kv_pair = @db.hgetall(@items).find { |_, v| v == item}
-        kv_pair.first unless kv_pair.nil?
-      end
     end
   end
 end

data/lib/redis/autosuggest/config.rb

@@ -3,46 +3,55 @@ class Redis
 
     # Default Redis server at localhost:6379
     @redis = Redis.new
+    
+    # Main Redis namespace for this module
+    @namespace = "suggest"
 
-    @db = Redis::Namespace.new("autosuggest", :redis => @redis)
+    @db = Redis::Namespace.new(@namespace, :redis => @redis)
 
     # Key for a Redis hash mapping ids to items we want to use for autosuggest responses
     @items = "items"
 
+    # Key to a Redis hash mapping items to their respective ids
+    @itemids = "itemids"
+
     # If we want to autosuggest for partial matchings of the word: 'ruby', we would
     # have four sorted sets: 'autosuggest:substring:r', 'autosuggest:substring:ru',
     # 'autosuggest:substring:rub', and 'autosuggest:substring:ruby'.
     # Each sorted set would the id to the word 'ruby'
-    @substrings =  Redis::Namespace.new("autosuggest:substring", :redis => @redis)
+    @substrings =  Redis::Namespace.new("#{@namespace}:sub", :redis => @redis)
 
     # max number of ids to store per substring.
     @max_per_substring = Float::INFINITY
 
     # max number of results to return for an autosuggest query
-    @max_results = 5 
+    @max_results = 5
 
     # Key to a sorted set holding all id of items in the autosuggest database sorted 
     # by their score
     @leaderboard = "lead"
 
-    # Leaderboard off by default
+    # Leaderboard on by default
     @use_leaderboard = false
 
     # Sources to be used for Autocomplete in rails.
     # Example: { Movie => :movie_title }
     @rails_sources = {}
 
+    # Stores the number of items the db has for each rails source
+    @rails_source_sizes = Redis::Namespace.new("#{@namespace}:size", :redis => @redis)
+
     class << self
       attr_reader :redis
-      attr_accessor :db, :items, :substrings, :max_per_substring, :max_results,
-        :leaderboard, :use_leaderboard, :rails_sources
+      attr_accessor :namespace, :db, :items, :substrings, :max_per_substring, :max_results,
+        :leaderboard, :use_leaderboard, :rails_sources, :rails_source_sizes
 
       def redis=(redis)
         @redis = redis
-        @db = Redis::Namespace.new("autosuggest", :redis => redis)
-        @substrings =  Redis::Namespace.new("autosuggest:substring", :redis => redis)
+        @db = Redis::Namespace.new(@namespace, :redis => redis)
+        @substrings =  Redis::Namespace.new("#{@namespace}:sub", :redis => redis)
+        @rails_source_sizes = Redis::Namespace.new("#{@namespace}:size", :redis => redis)
       end
     end
   end
 end
-

data/lib/redis/autosuggest/rails/railtie.rb

@@ -0,0 +1,10 @@
+class Redis
+  module Autosuggest
+    class Railtie < Rails::Railtie
+
+      rake_tasks do
+        load File.expand_path('../rake_tasks.rb', __FILE__)
+      end
+    end
+  end
+end

data/lib/redis/autosuggest/rails/rake_tasks.rb

@@ -0,0 +1,9 @@
+require "redis-autosuggest"
+
+namespace :autosuggest do
+  
+  desc "redis autosuggestions init"
+  task :init => :environment do
+    Redis::Autosuggest::SuggestRails.init_rails_sources
+  end
+end

data/lib/redis/autosuggest/rails/sources.rb

@@ -0,0 +1,100 @@
+class Redis
+  module Autosuggest
+    extend ActiveSupport::Concern
+    module ClassMethods
+
+      def autosuggest(column, options={})
+        hash = Redis::Autosuggest.rails_sources[self]
+        if hash.nil?
+          Redis::Autosuggest.rails_sources[self] = { column => options }
+        else
+          hash[column] = options
+        end
+
+        # Hook onto rails callbacks to update autosuggest db if a source is modified
+        class_eval <<-HERE
+          after_create :add_to_autosuggest
+          def add_to_autosuggest
+            Redis::Autosuggest::SuggestRails.add_to_autosuggest(self)
+          end
+
+          after_update :check_if_changed
+          def check_if_changed
+            Redis::Autosuggest::SuggestRails.check_if_changed(self)
+          end
+
+          before_destroy :remove_from_autosuggest
+          def remove_from_autosuggest
+            Redis::Autosuggest::SuggestRails.remove_from_autosuggest(self)
+          end
+          HERE
+        end
+      end
+
+      module SuggestRails
+        class << self
+
+          def init_rails_sources
+            Rails.application.eager_load!
+            Redis::Autosuggest.db.flushdb
+            Redis::Autosuggest.rails_sources.each do |model, attrs|
+              attrs.each do |column, options|
+                order = options[:init_order] || ""
+                model.order(order).find_each do |record|
+                  puts "Adding #{record.send(column)}"
+                  size = self.add_source_attr(record, column, options)
+                  break if size >= options[:limit]
+                end
+              end
+            end
+          end
+
+          def add_to_autosuggest(record)
+            Redis::Autosuggest.rails_sources[record.class].each do |column, options|
+              self.add_source_attr(record, column, options)
+            end
+          end
+
+          def add_source_attr(record, column, options)
+            item = record.send(column)
+            size = self.get_size(record.class, column).to_i
+            if size < options[:limit]
+              score = record.send(options[:rank_by]) unless options[:rank_by].nil?
+              score ||= 0
+              is_new_item = Redis::Autosuggest.add_with_score(item, score)
+              size = self.incr_size(record.class, column) if is_new_item
+            end
+            return size
+          end
+
+          def check_if_changed(record)
+            Redis::Autosuggest.rails_sources[record.class].each_key do |column|
+              next unless record.send("#{column}_changed?")
+              old_item = record.send("#{column}_was")
+              score = Redis::Autosuggest.get_score(old_item)
+              Redis::Autosuggest.remove(old_item)
+              Redis::Autosuggest.add_with_score(record.send(column), score)
+            end
+          end
+
+          def remove_from_autosuggest(record)
+            Redis::Autosuggest.rails_sources[record.class].each_key do |column|
+              item = record.send(column)
+              item_was_in_db = Redis::Autosuggest.remove(item)
+              self.incr_size(record.class, column, -1) if item_was_in_db
+            end
+          end
+
+          # Get the size (how many items) of a model/attribute pair
+          def get_size(model, attr)
+            Redis::Autosuggest.rails_source_sizes.get("#{model}:#{attr}")
+          end
+
+          # Increment the key storing the size of a model/attribute pair
+          def incr_size(model, attr, incr=1)
+            return Redis::Autosuggest.rails_source_sizes.incrby("#{model}:#{attr}", incr)
+          end
+        end
+      end
+    end
+  end

data/lib/redis/autosuggest/version.rb

@@ -1,5 +1,5 @@
 class Redis
   module Autosuggest
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/redis-autosuggest.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency("redis", "~> 3.0.1")
+  gem.add_dependency("redis", "~> 3.0.2")
   gem.add_dependency("redis-namespace", "~> 1.2.1")
 
-  gem.add_development_dependency("minitest", "~> 3.5.0")
+  gem.add_development_dependency("minitest", "~> 4.3.3")
 end

data/test/autosuggest_test.rb

@@ -1,7 +1,7 @@
 require 'test_helper'
 
 class TestAutosuggest < MiniTest::Unit::TestCase
-  
+
   def self.unused_db
     @unused_db ||= Redis.new(:db => TestHelper.db_picker)
   end
@@ -44,7 +44,7 @@ class TestAutosuggest < MiniTest::Unit::TestCase
     assert @subs.keys.size == 10
   end
 
-    def test_adding_multiple_items_with_scores
+  def test_adding_multiple_items_with_scores
     Redis::Autosuggest.add_with_score("one", 1, "two", 2, "three", 3)
     assert @db.hgetall(Redis::Autosuggest.items).size == 3
     assert @subs.keys.size == 10
@@ -101,8 +101,16 @@ class TestAutosuggest < MiniTest::Unit::TestCase
   end
 
   def test_getting_an_items_score
-     Redis::Autosuggest.add_with_score(@str1, 3)
-     assert_equal 3, Redis::Autosuggest.get_score(@str1)
+    Redis::Autosuggest.add_with_score(@str1, 3)
+    assert_equal 3, Redis::Autosuggest.get_score(@str1)
+  end
+
+  def test_adding_with_substring_limit
+    Redis::Autosuggest.max_per_substring = 1
+    Redis::Autosuggest.add_with_score(@str1, 1)
+    Redis::Autosuggest.add_with_score("Test", 5)
+    item_id = Redis::Autosuggest.get_id("Test")
+    assert_equal [item_id], @subs.zrevrange("test", 0, -1)
   end
 
   MiniTest::Unit.after_tests { self.unused_db.flushdb }

data/test/file_test.rb

@@ -30,4 +30,3 @@ class TestFile < MiniTest::Unit::TestCase
 
   MiniTest::Unit.after_tests { self.unused_db.flushdb }
 end
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis-autosuggest
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-26 00:00:00.000000000 Z
+date: 2012-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.0.2
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 4.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 4.3.3
 description: ! "Provides autocompletions through Redis, with the ability to rank\n
   \ results and integrate with Rails"
 email:
@@ -76,8 +76,9 @@ files:
 - lib/redis/autosuggest.rb
 - lib/redis/autosuggest/config.rb
 - lib/redis/autosuggest/file.rb
-- lib/redis/autosuggest/init.rb
-- lib/redis/autosuggest/rails.rb
+- lib/redis/autosuggest/rails/railtie.rb
+- lib/redis/autosuggest/rails/rake_tasks.rb
+- lib/redis/autosuggest/rails/sources.rb
 - lib/redis/autosuggest/version.rb
 - redis-autosuggest.gemspec
 - test/autosuggest_test.rb

data/lib/redis/autosuggest/init.rb

@@ -1,17 +0,0 @@
-class Redis
-  module Autosuggest
-
-    class << self
-
-      def init_rails_sources
-        self.rails_sources.each_key do |r|
-          r.all.each do |record|
-            record.add_to_autosuggest
-          end
-        end
-      end
-    end
-  end
-end
-
-

data/lib/redis/autosuggest/rails.rb

@@ -1,49 +0,0 @@
-class Redis
-  module Autosuggest
-    extend ActiveSupport::Concern
-
-    module ClassMethods
-
-      def autosuggest(column, options={})
-        hash = Redis::Autosuggest.rails_sources[self]
-        if hash.nil?
-          Redis::Autosuggest.rails_sources[self] = { column => options }
-        else
-          hash[column] = options
-        end
-
-        # hook onto rails callbacks to update autosuggest db if a source is modified
-        class_eval <<-HERE
-          after_create :add_to_autosuggest
-          def add_to_autosuggest
-            Redis::Autosuggest.rails_sources[self.class].each do |column, options|
-              score = self.send(options[:rank_by]) if !options[:rank_by].nil?
-              score ||= 0
-              Redis::Autosuggest.add_with_score(self.send(column), score)
-            end
-          end
-
-          after_update :check_if_changed
-          def check_if_changed
-            Redis::Autosuggest.rails_sources[self.class].each_key do |column|
-              next if !self.send("#{column}_changed?")
-              old_item = self.send("#{column}_was")
-              score = Redis::Autosuggest.get_score(old_item)
-              Redis::Autosuggest.remove(old_item)
-              Redis::Autosuggest.add_with_score(self.send(column), score)
-            end
-          end
-
-          before_destroy :remove_from_autosuggest
-          def remove_from_autosuggest
-            Redis::Autosuggest.rails_sources[self.class].each_key do |column|
-              Redis::Autosuggest.remove(self.send(column))
-            end
-          end
-          HERE
-        end 
-      end
-    end
-  end
-
-