soulmate 0.0.6 → 0.1.0

data/README.markdown

@@ -110,6 +110,9 @@ Add this to gemfile:
 
 Then you can query soulmate at the /sm url, for example: http://localhost:3000/sm/search?types[]=venues&limit=6&term=kitten
 
+### Rendering an autocompleter
+
+Soulmate doesn't include any client-side code necessary to render an autocompleter, but Mitch Crowe put together a pretty cool looking jquery plugin designed for exactly that: <a href="https://github.com/mcrowe/soulmate.js">soulmate.js</a>.
 
 Contributing to soulmate
 ------------------------

data/bin/soulmate

@@ -20,6 +20,12 @@ parser = OptionParser.new do |opts|
     Soulmate.redis = host
   end
 
+  opts.on("-s", "--stop-words [FILE]", "Path to file containing a list of stop words") do |fn|
+    File.open(fn) do |file|
+      Soulmate.stop_words = file.readlines.map{ |l| l.strip }
+    end
+  end
+
   opts.on("-h", "--help", "Show this message") do
     puts opts
     exit

data/lib/soulmate.rb

@@ -13,7 +13,7 @@ module Soulmate
   extend self
 
   MIN_COMPLETE = 2
-  STOP_WORDS = ["vs", "at"]
+  DEFAULT_STOP_WORDS = ["vs", "at", "the"]
 
   def redis=(url)
     @redis = nil
@@ -34,4 +34,12 @@ module Soulmate
     )
   end
 
+  def stop_words
+    @stop_words ||= DEFAULT_STOP_WORDS
+  end
+
+  def stop_words=(arr)
+    @stop_words = Array(arr).flatten
+  end
+
 end

data/lib/soulmate/helpers.rb

@@ -2,7 +2,9 @@ module Soulmate
   module Helpers
 
     def prefixes_for_phrase(phrase)
-      words = normalize(phrase).split(' ')
+      words = normalize(phrase).split(' ').reject do |w|
+        Soulmate.stop_words.include?(w)
+      end
       words.map do |w|
         (MIN_COMPLETE-1..(w.length-1)).map{ |l| w[0..l] }
       end.flatten.uniq

data/lib/soulmate/loader.rb

@@ -4,12 +4,13 @@ module Soulmate
 
     def load(items)
       # delete the sorted sets for this type
-      # wrap in multi/exec?
       phrases = Soulmate.redis.smembers(base)
-      phrases.each do |p|
-        Soulmate.redis.del("#{base}:#{p}")
+      Soulmate.redis.pipelined do
+        phrases.each do |p|
+          Soulmate.redis.del("#{base}:#{p}")
+        end
+        Soulmate.redis.del(base)
       end
-      Soulmate.redis.del(base)
 
       # Redis can continue serving cached requests for this type while the reload is
       # occuring. Some requests may be cached incorrectly as empty set (for requests
@@ -32,12 +33,14 @@ module Soulmate
       # kill any old items with this id
       remove("id" => item["id"]) unless opts[:skip_duplicate_check]
       
-      # store the raw data in a separate key to reduce memory usage
-      Soulmate.redis.hset(database, item["id"], MultiJson.encode(item))
-      phrase = ([item["term"]] + (item["aliases"] || [])).join(' ')
-      prefixes_for_phrase(phrase).each do |p|
-        Soulmate.redis.sadd(base, p) # remember this prefix in a master set
-        Soulmate.redis.zadd("#{base}:#{p}", item["score"], item["id"]) # store the id of this term in the index
+      Soulmate.redis.pipelined do
+        # store the raw data in a separate key to reduce memory usage
+        Soulmate.redis.hset(database, item["id"], MultiJson.encode(item))
+        phrase = ([item["term"]] + (item["aliases"] || [])).join(' ')
+        prefixes_for_phrase(phrase).each do |p|
+          Soulmate.redis.sadd(base, p) # remember this prefix in a master set
+          Soulmate.redis.zadd("#{base}:#{p}", item["score"], item["id"]) # store the id of this term in the index
+        end
       end
     end
 
@@ -47,11 +50,13 @@ module Soulmate
       if prev_item
         prev_item = MultiJson.decode(prev_item)
         # undo the operations done in add
-        Soulmate.redis.hdel(database, prev_item["id"])
-        phrase = ([prev_item["term"]] + (prev_item["aliases"] || [])).join(' ')
-        prefixes_for_phrase(phrase).each do |p|
-          Soulmate.redis.srem(base, p)
-          Soulmate.redis.zrem("#{base}:#{p}", prev_item["id"])
+        Soulmate.redis.pipelined do
+          Soulmate.redis.hdel(database, prev_item["id"])
+          phrase = ([prev_item["term"]] + (prev_item["aliases"] || [])).join(' ')
+          prefixes_for_phrase(phrase).each do |p|
+            Soulmate.redis.srem(base, p)
+            Soulmate.redis.zrem("#{base}:#{p}", prev_item["id"])
+          end
         end
       end
     end

data/lib/soulmate/matcher.rb

@@ -5,9 +5,7 @@ module Soulmate
     def matches_for_term(term, options = {})
       options = { :limit => 5, :cache => true }.merge(options)
       
-      words = normalize(term).split(' ').reject do |w|
-        w.size < MIN_COMPLETE or STOP_WORDS.include?(w)
-      end.sort
+      words = normalize(term).split(' ').reject{ |w| w.size < MIN_COMPLETE }.sort
 
       return [] if words.empty?
 

data/lib/soulmate/version.rb

@@ -1,8 +1,8 @@
 module Soulmate
   module Version
     MAJOR = 0
-    MINOR = 0
-    PATCH = 6
+    MINOR = 1
+    PATCH = 0
 
     STRING = [MAJOR, MINOR, PATCH].compact.join('.')
   end

data/soulmate.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{soulmate}
-  s.version = "0.0.6"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Eric Waller"]
-  s.date = %q{2011-11-18}
+  s.date = %q{2012-01-05}
   s.description = %q{Soulmate is a tool to help solve the common problem of developing a fast autocomplete feature. It uses Redis's sorted sets to build an index of partial words and corresponding top matches, and provides a simple sinatra app to query them. Soulmate finishes your sentences.}
   s.email = %q{eric@seatgeek.com}
   s.executables = ["soulmate", "soulmate-web"]
@@ -36,6 +36,7 @@ Gem::Specification.new do |s|
     "soulmate.gemspec",
     "test/db/.gitkeep",
     "test/helper.rb",
+    "test/samples/stop-words.txt",
     "test/samples/venues.json",
     "test/test.conf",
     "test/test_soulmate.rb"

data/test/samples/stop-words.txt

@@ -0,0 +1,4 @@
+vs
+at
+the
+to

data/test/test_soulmate.rb

@@ -94,6 +94,10 @@ class TestSoulmate < Test::Unit::TestCase
   def test_prefixes_for_phrase
     loader = Soulmate::Loader.new('venues')
     
+    Soulmate.stop_words = ['the']
+    
+    assert_equal ["kn", "kni", "knic", "knick", "knicks"], loader.prefixes_for_phrase("the knicks")
+    assert_equal ["te", "tes", "test", "testi", "testin", "th", "thi", "this"], loader.prefixes_for_phrase("testin' this")
     assert_equal ["te", "tes", "test", "testi", "testin", "th", "thi", "this"], loader.prefixes_for_phrase("testin' this")
     assert_equal ["te", "tes", "test"], loader.prefixes_for_phrase("test test")
     assert_equal ["so", "sou", "soul", "soulm", "soulma", "soulmat", "soulmate"], loader.prefixes_for_phrase("SoUlmATE")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: soulmate
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-18 00:00:00.000000000 -05:00
+date: 2012-01-05 00:00:00.000000000 -05:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2165158440 !ruby/object:Gem::Requirement
+  requirement: &2160527080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: 2.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *2165158440
+  version_requirements: *2160527080
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &2165156820 !ruby/object:Gem::Requirement
+  requirement: &2160525840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -33,10 +33,10 @@ dependencies:
         version: 1.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *2165156820
+  version_requirements: *2160525840
 - !ruby/object:Gem::Dependency
   name: rack-contrib
-  requirement: &2165155380 !ruby/object:Gem::Requirement
+  requirement: &2160524600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165155380
+  version_requirements: *2160524600
 - !ruby/object:Gem::Dependency
   name: vegas
-  requirement: &2165152760 !ruby/object:Gem::Requirement
+  requirement: &2160523120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +55,10 @@ dependencies:
         version: 0.1.8
   type: :runtime
   prerelease: false
-  version_requirements: *2165152760
+  version_requirements: *2160523120
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &2165151760 !ruby/object:Gem::Requirement
+  requirement: &2160516060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,10 +66,10 @@ dependencies:
         version: 1.2.3
   type: :runtime
   prerelease: false
-  version_requirements: *2165151760
+  version_requirements: *2160516060
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2165151060 !ruby/object:Gem::Requirement
+  requirement: &2160514180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -77,10 +77,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2165151060
+  version_requirements: *2160514180
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2165150120 !ruby/object:Gem::Requirement
+  requirement: &2160512560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -88,10 +88,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2165150120
+  version_requirements: *2160512560
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2165149320 !ruby/object:Gem::Requirement
+  requirement: &2160510860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -99,10 +99,10 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *2165149320
+  version_requirements: *2160510860
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2165148620 !ruby/object:Gem::Requirement
+  requirement: &2160509180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -110,10 +110,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2165148620
+  version_requirements: *2160509180
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2165147860 !ruby/object:Gem::Requirement
+  requirement: &2160504040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -121,10 +121,10 @@ dependencies:
         version: '2.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165147860
+  version_requirements: *2160504040
 - !ruby/object:Gem::Dependency
   name: vegas
-  requirement: &2165146920 !ruby/object:Gem::Requirement
+  requirement: &2160500820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -132,10 +132,10 @@ dependencies:
         version: 0.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *2165146920
+  version_requirements: *2160500820
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &2165145580 !ruby/object:Gem::Requirement
+  requirement: &2160499040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -143,10 +143,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165145580
+  version_requirements: *2160499040
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &2165137880 !ruby/object:Gem::Requirement
+  requirement: &2160480560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -154,7 +154,7 @@ dependencies:
         version: 1.4.6
   type: :runtime
   prerelease: false
-  version_requirements: *2165137880
+  version_requirements: *2160480560
 description: Soulmate is a tool to help solve the common problem of developing a fast
   autocomplete feature. It uses Redis's sorted sets to build an index of partial words
   and corresponding top matches, and provides a simple sinatra app to query them.
@@ -186,6 +186,7 @@ files:
 - soulmate.gemspec
 - test/db/.gitkeep
 - test/helper.rb
+- test/samples/stop-words.txt
 - test/samples/venues.json
 - test/test.conf
 - test/test_soulmate.rb
@@ -205,7 +206,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4567717493668113734
+      hash: 1384397421883417859
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: