soulmate 0.0.4 → 0.0.5

data/Gemfile

@@ -13,6 +13,7 @@ group :development do
 end
 
 gem 'redis',        "~> 2.1.1"
+gem 'multi_json',   "~> 1.0.3"
 gem 'rack-contrib'
 gem 'vegas',        "~> 0.1.8"
 gem 'sinatra',      "~> 1.2.3"

data/Gemfile.lock

@@ -6,6 +6,7 @@ GEM
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
+    multi_json (1.0.3)
     rack (1.2.2)
     rack-contrib (1.1.0)
       rack (>= 0.9.1)
@@ -26,6 +27,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.0.0)
   jeweler (~> 1.5.2)
+  multi_json (~> 1.0.3)
   rack-contrib
   rcov
   redis (~> 2.1.1)

data/README.markdown

@@ -56,7 +56,7 @@ Once it's loaded, we can query this data by starting `soulmate-web`:
 
     $ soulmate-web --foreground --no-launch --redis=redis://localhost:6379/0
 
-And viewing the service in your browser: <a href="http://localhost:5678/search?types[]=venue&term=stad">http://localhost:5678/search?types[]=venue&term=stad</a>. You should see something like:
+And viewing the service in your browser: http://localhost:5678/search?types[]=venue&term=stad. You should see something like:
 
     {
       "term": "stad",

data/bin/soulmate

@@ -27,14 +27,47 @@ parser = OptionParser.new do |opts|
 
   opts.separator ""
   opts.separator "Commands:"
-  opts.separator "  load TYPE   Loads items of specified type read from stdin in the JSON lines format"
+  opts.separator "  load   TYPE        Replaces collection specified by TYPE with items read from stdin in the JSON lines format."
+  opts.separator "  add    TYPE        Adds items to collection specified by TYPE read from stdin in the JSON lines format."
+  opts.separator "  remove TYPE        Removes items from collection specified by TYPE read from stdin in the JSON lines format. Items only require an 'id', all other fields are ignored."
+  opts.separator "  query  TYPE QUERY  Queries for items from collection specified by TYPE."
 end
 
 def load(type)
   puts "Loading items of type #{type}..."
-  items = $stdin.read.split("\n").map { |l| JSON.parse(l) }
-  total = Soulmate::Loader.new(type).load(items)
-  puts "Loaded a total of #{total} items"
+  items = $stdin.read.split("\n").map { |l| MultiJson.decode(l) }
+  loaded = Soulmate::Loader.new(type).load(items)
+  puts "Loaded a total of #{loaded.size} items"
+end
+
+def add(type)
+  puts "Adding items of type #{type}..."
+  loader = Soulmate::Loader.new(type)
+  items = $stdin.read.split("\n").map { |l| MultiJson.decode(l) }
+  items.each do |item|
+    loader.add(item)
+  end
+  puts "Loaded a total of #{items.size} items"
+end
+
+def remove(type)
+  puts "Removing items of type #{type}..."
+  loader = Soulmate::Loader.new(type)
+  items = $stdin.read.split("\n").map { |l| MultiJson.decode(l) }
+  items.each do |item|
+    loader.remove(item)
+  end
+  puts "Removed a total of #{items.size} items"
+end
+
+def query(type, query)
+  puts "> Querying '#{type}' for '#{query}'"
+  matcher = Soulmate::Matcher.new(type)
+  results = matcher.matches_for_term(query, :limit => 0)
+  results.each do |item|
+    puts MultiJson.encode(item)
+  end
+  puts "> Found #{results.size} matches"
 end
 
 parser.parse!
@@ -42,6 +75,12 @@ parser.parse!
 case ARGV[0]
 when 'load'
   load ARGV[1]
+when 'add'
+  add ARGV[1]
+when 'remove'
+  remove ARGV[1]
+when 'query'
+  query ARGV[1], ARGV[2]
 else
   puts parser.help
 end

data/lib/soulmate.rb

@@ -1,5 +1,5 @@
 require 'uri'
-require 'json'
+require 'multi_json'
 require 'redis'
 
 require 'soulmate/version'

data/lib/soulmate/helpers.rb

@@ -5,7 +5,7 @@ module Soulmate
       words = normalize(phrase).split(' ')
       words.map do |w|
         (MIN_COMPLETE-1..(w.length-1)).map{ |l| w[0..l] }
-      end.flatten
+      end.flatten.uniq
     end
 
     def normalize(str)

data/lib/soulmate/loader.rb

@@ -19,27 +19,41 @@ module Soulmate
       # delete the data stored for this type
       Soulmate.redis.del(database)
 
-      items_loaded = 0
       items.each_with_index do |item, i|
-        id    = item["id"]
-        term  = item["term"]
-        score = item["score"]
+        add(item, :skip_duplicate_check => true)
+      end
+    end
 
-        if id and term
-          # store the raw data in a separate key to reduce memory usage
-          Soulmate.redis.hset(database, id, JSON.dump(item))
+    # "id", "term", "score", "aliases", "data"
+    def add(item, opts = {})
+      opts = { :skip_duplicate_check => false }.merge(opts)
+      raise ArgumentError unless item["id"] && item["term"]
+      
+      # kill any old items with this id
+      remove("id" => item["id"]) unless opts[:skip_duplicate_check]
+      
+      # store the raw data in a separate key to reduce memory usage
+      Soulmate.redis.hset(database, item["id"], MultiJson.encode(item))
+      phrase = ([item["term"]] + (item["aliases"] || [])).join(' ')
+      prefixes_for_phrase(phrase).each do |p|
+        Soulmate.redis.sadd(base, p) # remember this prefix in a master set
+        Soulmate.redis.zadd("#{base}:#{p}", item["score"], item["id"]) # store the id of this term in the index
+      end
+    end
 
-          phrase = ([term] + (item["aliases"] || [])).join(' ')
-          prefixes_for_phrase(phrase).uniq.each do |p|
-            Soulmate.redis.sadd(base, p) # remember this prefix in a master set
-            Soulmate.redis.zadd("#{base}:#{p}", score, id) # store the id of this term in the index
-          end
-          items_loaded += 1
+    # remove only cares about an item's id, but for consistency takes an object
+    def remove(item)
+      prev_item = Soulmate.redis.hget(database, item["id"])
+      if prev_item
+        prev_item = MultiJson.decode(prev_item)
+        # undo the operations done in add
+        Soulmate.redis.hdel(database, prev_item["id"])
+        phrase = ([prev_item["term"]] + (prev_item["aliases"] || [])).join(' ')
+        prefixes_for_phrase(phrase).each do |p|
+          Soulmate.redis.srem(base, p)
+          Soulmate.redis.zrem("#{base}:#{p}", prev_item["id"])
         end
-        puts "added #{i} entries" if i % 100 == 0 and i != 0
       end
-
-      items_loaded
     end
   end
 end

data/lib/soulmate/matcher.rb

@@ -3,22 +3,30 @@ module Soulmate
   class Matcher < Base
 
     def matches_for_term(term, options = {})
+      options = { :limit => 5, :cache => true }.merge(options)
+      
       words = normalize(term).split(' ').reject do |w|
         w.size < MIN_COMPLETE or STOP_WORDS.include?(w)
       end.sort
 
-      options[:limit] ||= 5
+      return [] if words.empty?
 
       cachekey = "#{cachebase}:" + words.join('|')
 
-      if !Soulmate.redis.exists(cachekey)
+      if !options[:cache] || !Soulmate.redis.exists(cachekey)
         interkeys = words.map { |w| "#{base}:#{w}" }
         Soulmate.redis.zinterstore(cachekey, interkeys)
         Soulmate.redis.expire(cachekey, 10 * 60) # expire after 10 minutes
       end
 
       ids = Soulmate.redis.zrevrange(cachekey, 0, options[:limit] - 1)
-      ids.size > 0 ? Soulmate.redis.hmget(database, *ids).map { |r| JSON.parse(r) } : []
+      if ids.size > 0
+        Soulmate.redis.hmget(database, *ids)
+          .reject{ |r| r.nil? } # handle cached results for ids which have since been deleted
+          .map { |r| MultiJson.decode(r) }
+      else
+        []
+      end
     end
   end
 end

data/lib/soulmate/server.rb

@@ -14,7 +14,7 @@ module Soulmate
     end
     
     get '/' do
-      JSON.pretty_generate({ :soulmate => Soulmate::Version::STRING, :status   => "ok" })
+      MultiJson.encode({ :soulmate => Soulmate::Version::STRING, :status   => "ok" })
     end
     
     get '/search' do
@@ -30,7 +30,7 @@ module Soulmate
         results[type] = matcher.matches_for_term(term, :limit => limit)
       end
       
-      JSON.pretty_generate({
+      MultiJson.encode({
         :term    => params[:term],
         :results => results
       })
@@ -38,7 +38,7 @@ module Soulmate
     
     not_found do
       content_type 'application/json', :charset => 'utf-8'
-      JSON.pretty_generate({ :error => "not found" })
+      MultiJson.encode({ :error => "not found" })
     end
     
   end

data/lib/soulmate/version.rb

@@ -2,7 +2,7 @@ module Soulmate
   module Version
     MAJOR = 0
     MINOR = 0
-    PATCH = 4
+    PATCH = 5
 
     STRING = [MAJOR, MINOR, PATCH].compact.join('.')
   end

data/soulmate.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{soulmate}
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Eric Waller"]
-  s.date = %q{2011-07-07}
+  s.date = %q{2011-10-12}
   s.description = %q{Soulmate is a tool to help solve the common problem of developing a fast autocomplete feature. It uses Redis's sorted sets to build an index of partial words and corresponding top matches, and provides a simple sinatra app to query them. Soulmate finishes your sentences.}
   s.email = %q{eric@seatgeek.com}
   s.executables = ["soulmate", "soulmate-web"]
@@ -55,6 +55,7 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<redis>, ["~> 2.1.1"])
+      s.add_runtime_dependency(%q<multi_json>, ["~> 1.0.3"])
       s.add_runtime_dependency(%q<rack-contrib>, [">= 0"])
       s.add_runtime_dependency(%q<vegas>, ["~> 0.1.8"])
       s.add_runtime_dependency(%q<sinatra>, ["~> 1.2.3"])
@@ -68,6 +69,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<json>, ["~> 1.4.6"])
     else
       s.add_dependency(%q<redis>, ["~> 2.1.1"])
+      s.add_dependency(%q<multi_json>, ["~> 1.0.3"])
       s.add_dependency(%q<rack-contrib>, [">= 0"])
       s.add_dependency(%q<vegas>, ["~> 0.1.8"])
       s.add_dependency(%q<sinatra>, ["~> 1.2.3"])
@@ -82,6 +84,7 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<redis>, ["~> 2.1.1"])
+    s.add_dependency(%q<multi_json>, ["~> 1.0.3"])
     s.add_dependency(%q<rack-contrib>, [">= 0"])
     s.add_dependency(%q<vegas>, ["~> 0.1.8"])
     s.add_dependency(%q<sinatra>, ["~> 1.2.3"])

data/test/test_soulmate.rb

@@ -5,12 +5,12 @@ class TestSoulmate < Test::Unit::TestCase
     items = []
     venues = File.open(File.expand_path(File.dirname(__FILE__)) + '/samples/venues.json', "r")
     venues.each_line do |venue|
-      items << JSON.parse(venue)
+      items << MultiJson.decode(venue)
     end
     
     items_loaded = Soulmate::Loader.new('venues').load(items)
     
-    assert_equal 6, items_loaded
+    assert_equal 6, items_loaded.size
     
     matcher = Soulmate::Matcher.new('venues')
     results = matcher.matches_for_term('stad', :limit => 5)
@@ -23,12 +23,12 @@ class TestSoulmate < Test::Unit::TestCase
     items = []
     venues = File.open(File.expand_path(File.dirname(__FILE__)) + '/samples/venues.json', "r")
     venues.each_line do |venue|
-      items << JSON.parse(venue)
+      items << MultiJson.decode(venue)
     end
     
     items_loaded = Soulmate::Loader.new('venues').load(items)
     
-    assert_equal 6, items_loaded
+    assert_equal 6, items_loaded.size
     
     matcher = Soulmate::Matcher.new('venues')
     results = matcher.matches_for_term('land shark stadium', :limit => 5)
@@ -42,4 +42,60 @@ class TestSoulmate < Test::Unit::TestCase
     results = matcher.matches_for_term('stadium', :limit => 5)    
     assert_equal 5, results.size
   end
+  
+  def test_can_remove_items
+    
+    loader = Soulmate::Loader.new('venues')
+    matcher = Soulmate::Matcher.new('venues')
+    
+    # empty the collection
+    loader.load([])
+    results = matcher.matches_for_term("te", :cache => false)
+    assert_equal 0, results.size
+    
+    loader.add("id" => 1, "term" => "Testing this", "score" => 10)
+    results = matcher.matches_for_term("te", :cache => false)
+    assert_equal 1, results.size
+    
+    loader.remove("id" => 1)
+    results = matcher.matches_for_term("te", :cache => false)
+    assert_equal 0, results.size
+    
+  end
+  
+  def test_can_update_items
+    
+    loader = Soulmate::Loader.new('venues')
+    matcher = Soulmate::Matcher.new('venues')
+    
+    # empty the collection
+    loader.load([])
+    
+    # initial data
+    loader.add("id" => 1, "term" => "Testing this", "score" => 10)
+    loader.add("id" => 2, "term" => "Another Term", "score" => 9)
+    loader.add("id" => 3, "term" => "Something different", "score" => 5)
+    
+    results = matcher.matches_for_term("te", :cache => false)
+    assert_equal 2, results.size
+    assert_equal "Testing this", results.first["term"]
+    assert_equal 10, results.first["score"]
+    
+    # update id:1
+    loader.add("id" => 1, "term" => "Updated", "score" => 5)
+    
+    results = matcher.matches_for_term("te", :cache => false)
+    assert_equal 1, results.size
+    assert_equal "Another Term", results.first["term"]
+    assert_equal 9, results.first["score"]
+    
+  end
+  
+  def test_prefixes_for_phrase
+    loader = Soulmate::Loader.new('venues')
+    
+    assert_equal ["te", "tes", "test", "testi", "testin", "th", "thi", "this"], loader.prefixes_for_phrase("testin' this")
+    assert_equal ["te", "tes", "test"], loader.prefixes_for_phrase("test test")
+    assert_equal ["so", "sou", "soul", "soulm", "soulma", "soulmat", "soulmate"], loader.prefixes_for_phrase("SoUlmATE")
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: soulmate
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-07-07 00:00:00.000000000 -04:00
+date: 2011-10-12 00:00:00.000000000 -04:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2164813260 !ruby/object:Gem::Requirement
+  requirement: &2151919160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,21 @@ dependencies:
         version: 2.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *2164813260
+  version_requirements: *2151919160
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &2151917560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *2151917560
 - !ruby/object:Gem::Dependency
   name: rack-contrib
-  requirement: &2164811620 !ruby/object:Gem::Requirement
+  requirement: &2151916600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +44,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2164811620
+  version_requirements: *2151916600
 - !ruby/object:Gem::Dependency
   name: vegas
-  requirement: &2164810860 !ruby/object:Gem::Requirement
+  requirement: &2151915480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,10 +55,10 @@ dependencies:
         version: 0.1.8
   type: :runtime
   prerelease: false
-  version_requirements: *2164810860
+  version_requirements: *2151915480
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &2164809840 !ruby/object:Gem::Requirement
+  requirement: &2151914480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +66,10 @@ dependencies:
         version: 1.2.3
   type: :runtime
   prerelease: false
-  version_requirements: *2164809840
+  version_requirements: *2151914480
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2164808840 !ruby/object:Gem::Requirement
+  requirement: &2151913720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,10 +77,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2164808840
+  version_requirements: *2151913720
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2164807780 !ruby/object:Gem::Requirement
+  requirement: &2151912740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -77,10 +88,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2164807780
+  version_requirements: *2151912740
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2164807160 !ruby/object:Gem::Requirement
+  requirement: &2151911840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -88,10 +99,10 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *2164807160
+  version_requirements: *2151911840
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2164806540 !ruby/object:Gem::Requirement
+  requirement: &2151910900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -99,10 +110,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2164806540
+  version_requirements: *2151910900
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2164805780 !ruby/object:Gem::Requirement
+  requirement: &2151909380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -110,10 +121,10 @@ dependencies:
         version: '2.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2164805780
+  version_requirements: *2151909380
 - !ruby/object:Gem::Dependency
   name: vegas
-  requirement: &2164805020 !ruby/object:Gem::Requirement
+  requirement: &2151907400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -121,10 +132,10 @@ dependencies:
         version: 0.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *2164805020
+  version_requirements: *2151907400
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &2164804440 !ruby/object:Gem::Requirement
+  requirement: &2151904740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -132,10 +143,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2164804440
+  version_requirements: *2151904740
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &2164803800 !ruby/object:Gem::Requirement
+  requirement: &2151901600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -143,7 +154,7 @@ dependencies:
         version: 1.4.6
   type: :runtime
   prerelease: false
-  version_requirements: *2164803800
+  version_requirements: *2151901600
 description: Soulmate is a tool to help solve the common problem of developing a fast
   autocomplete feature. It uses Redis's sorted sets to build an index of partial words
   and corresponding top matches, and provides a simple sinatra app to query them.
@@ -194,7 +205,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1504170323034549202
+      hash: -3865399172879930147
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: