lunar 0.4.1 → 0.5.0

data/lib/lunar/keyword_matches.rb

@@ -0,0 +1,32 @@
+module Lunar
+  # @private Used internally by Lunar::search to get all the word matches
+  # given `nest`, `att` and it's `val`.
+  class KeywordMatches
+    attr :nest
+    attr :att
+    attr :value
+
+    def initialize(nest, att, value)
+      @nest, @att, @value = nest, att.to_sym, value
+    end
+
+    def distkey
+      nest[{ att => value }.hash].tap do |dk|
+        dk.zunionstore keys.flatten
+      end
+    end
+
+  protected
+    def keys
+      if att == :q
+        metaphones.map { |m| nest['*'][m].keys }
+      else
+        metaphones.map { |m| nest[att][m] }
+      end
+    end
+
+    def metaphones
+      Words.new(value).map { |word| Lunar.metaphone(word) }
+    end
+  end
+end

data/lib/lunar/lunar_nest.rb

@@ -0,0 +1,19 @@
+module Lunar
+  # @private Provides convenience methods to look up keys.
+  # Since the Redis KEYS command is actually pretty slow,
+  # i'm considering an alternative approach of manually maintaining
+  # the sets to manage all the groups of keys.
+  class LunarNest < Nest
+    def keys
+      redis.keys self
+    end
+
+    def matches
+      regex = Regexp.new(self.gsub('*', '(.*)'))
+      keys.map { |key|
+        match = key.match(regex)
+        [LunarNest.new(key, redis), *match[1, match.size - 1]]
+      }
+    end
+  end
+end

data/lib/lunar/range_matches.rb

@@ -0,0 +1,28 @@
+module Lunar
+  # @private Used internally by Lunar::search to get all the range matches
+  # given `nest`, `att` and it's `val`.
+  class RangeMatches
+    MAX_RESULTS = 10_000
+
+    attr :nest
+    attr :att
+    attr :range
+
+    def initialize(nest, att, range)
+      @nest, @att, @range = nest, att.to_sym, range
+    end
+
+
+    def distkey
+      nest[{ att => range }.hash].tap do |dk|
+        res = zset.zrangebyscore(range.first, range.last, :limit => [0, MAX_RESULTS])
+        res.each { |val| dk.zadd(1, val) }
+      end
+    end
+
+  private
+    def zset
+      @sortedset ||= nest[:Numbers][att]
+    end
+  end
+end

data/lib/lunar/result_set.rb

@@ -1,43 +1,100 @@
 module Lunar
+  # This wraps a Lunar search result set. You can do all the
+  # expected operations with an Enumerable.
+  #
+  # @example:
+  #
+  #   results = Lunar.search(Gadget, :q => "Apple Macbook Pro")
+  #   results.class == Lunar::ResultSet
+  #   # => true
+  #
+  #   results.kind_of?(Enumerable)
+  #   # => true
+  #
+  # `sort` and `sort_by` commands are available and directly calls
+  # the Redis SORT command.
+  #
+  #   results = Lunar.search(Gadget, :q => "Apple Macbook Pro")
+  #   results.sort(:by => :name, :order => "ALPHA")
+  #   results.sort(:by => :name, :order => "ALPHA ASC")
+  #   results.sort(:by => :name, :order => "ALPHA DESC")
+  #
+  # @see http://code.google.com/p/redis/wiki/SortCommand
   class ResultSet
     include Enumerable
 
-    attr :key
+    attr :distkey
+    attr :nest
+    attr :sortables
 
-    def initialize(key, &block)
-      @key   = key
-      @block = block
+    def initialize(distkey, nest, finder)
+      @distkey   = distkey
+      @nest      = nest
+      @finder    = finder
+      @sortables = @nest[:Sortables]['*']
     end
 
-    def each(&block)
-      all.each(&block)
-    end
-  end
-
-  class SortedResultSet < ResultSet
-    def all(options = {})
-      start  = Integer(options[:start] || 0)
-      limit  = Integer(options[:limit] || 0)
-      finish = start + limit - 1
-      
-      Lunar.redis.zrevrange(key, start, finish).map(&@block)
+    def each
+      objects(distkey.zrange(0, -1)).each { |e| yield e }
     end
 
-    def size
-      Lunar.redis.zcard(key)
+    # Provides syntatic sugar for `sort`.
+    #
+    # @example:
+    #
+    #   results = Lunar.search(Gadget, :q => "apple")
+    #   results.sort(:by => :votes)
+    #   results.sort_by(:votes)
+    #
+    # @param [#to_s] att the field in the namespace you want to sort by.
+    # @param [Hash] opts the various opts to pass to Redis SORT command.
+    # @option opts [#to_s] :order the direction you want to sort i.e. ASC DESC ALPHA
+    # @option opts [Array] :limit offset and max results to return.
+    #
+    # @return [Array] Array of objects as defined by the `finder`.
+    # @see http://code.google.com/p/redis/wiki/SortCommand
+    def sort_by(att, opts = {})
+      sort(opts.merge(:by => att))
     end
-  end
 
-  class UnsortedResultSet < ResultSet
-    def all(options = {})
-      start  = Integer(options[:start] || 0)
-      limit  = Integer(options[:limit] || 100)
-      
-      Lunar.redis.sort(key, :limit => [start, limit]).map(&@block)
+    # Gives the ability to sort the search results via a `sortable` field
+    # in your index.
+    #
+    # @example:
+    #
+    #   Lunar.index Gadget do |i|
+    #     i.id 1001
+    #     i.text :title, "Apple Macbook Pro"
+    #     i.sortable :votes, 10
+    #   end
+    #
+    #   Lunar.index Gadget do |i|
+    #     i.id 1002
+    #     i.text :title, "Apple iPad"
+    #     i.sortable :votes, 50
+    #   end
+    #
+    #   results = Lunar.search(Gadget, :q => "apple")
+    #   sorted  = results.sort(:by => :votes, :order => "DESC")
+    #
+    #   sorted == [Gadget[1002], Gadget[1001]]
+    #   # => true
+    #
+    # @param [Hash] opts the various opts to pass to Redis SORT command.
+    # @option opts [#to_s] :by the field in the namespace you want to sort by.
+    # @option opts [#to_s] :order the direction you want to sort i.e. ASC DESC ALPHA
+    # @option opts [Array] :limit offset and max results to return.
+    #
+    # @return [Array] Array of objects as defined by the `finder`.
+    # @see http://code.google.com/p/redis/wiki/SortCommand
+    def sort(opts = {})
+      opts[:by] = sortables[opts[:by]]  if opts[:by]
+      objects(distkey.sort(opts))
     end
 
-    def size
-      Lunar.redis.scard(key)
+  protected
+    def objects(ids)
+      ids.map(&@finder)
     end
   end
-end
+end

data/lib/lunar/scoring.rb

@@ -1,11 +1,13 @@
 module Lunar
+  # @private internally used by Lunar::Index to count the words
+  # of a given text.
   class Scoring
     def initialize(words)
       @words = Words.new(words)
     end
 
     def scores
-      @words.inject(Hash.new(0)) { |a, w| a[w.downcase] += 1; a }
+      @words.inject(Hash.new(0)) { |a, w| a[w.downcase] += 1 and a }
     end
   end
-end
+end

data/lib/lunar/stopwords.rb

@@ -0,0 +1,15 @@
+module Lunar
+  module Stopwords
+    def include?(word)
+      stopwords.include?(word)
+    end
+
+  private
+    def stopwords
+      %w(an and are as at be but by for if in into is it no not of on or s 
+         such t that the their then there these they this to was will with)
+    end
+
+    module_function :stopwords, :include?
+  end
+end

data/lib/lunar/words.rb

@@ -1,14 +1,17 @@
 require 'iconv'
 
 module Lunar
+  # @private Internally used to determine the words given some str.
+  # i.e. Words.new("the quick brown") == %w(the quick brown)
   class Words < Array
     SEPARATOR = /\s+/
 
     def initialize(str)
       words = str.split(SEPARATOR).
         reject { |w| w.to_s.strip.empty? }.
-        map    { |w| sanitize(w) }
-      
+        map    { |w| sanitize(w) }.
+        reject { |w| Stopwords.include?(w) }
+
       super(words)
     end
 
@@ -18,4 +21,4 @@ module Lunar
         gsub(/[^a-zA-Z0-9\-_]/, '').downcase
     end
   end
-end
+end

data/lunar.gemspec

@@ -5,92 +5,57 @@
 
 Gem::Specification.new do |s|
   s.name = %q{lunar}
-  s.version = "0.4.1"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Cyril David"]
-  s.date = %q{2010-05-20}
-  s.description = %q{uses sorted sets and sets, sorting by score}
+  s.date = %q{2010-05-21}
+  s.description = %q{Features full text searching via metaphones, range querying for numbers, fuzzy searching and sorting based on customer fields}
   s.email = %q{cyx.ucron@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.md"
+     "README.markdown"
   ]
   s.files = [
-    ".document",
-     ".gitignore",
-     "DATA",
+    ".gitignore",
      "LICENSE",
-     "README.md",
+     "README.markdown",
      "Rakefile",
      "VERSION",
-     "examples/ohm.rb",
      "lib/lunar.rb",
+     "lib/lunar/connection.rb",
+     "lib/lunar/fuzzy_matches.rb",
      "lib/lunar/fuzzy_word.rb",
      "lib/lunar/index.rb",
+     "lib/lunar/keyword_matches.rb",
+     "lib/lunar/lunar_nest.rb",
+     "lib/lunar/range_matches.rb",
      "lib/lunar/result_set.rb",
      "lib/lunar/scoring.rb",
-     "lib/lunar/search.rb",
-     "lib/lunar/sets.rb",
+     "lib/lunar/stopwords.rb",
      "lib/lunar/words.rb",
      "lunar.gemspec",
      "test/helper.rb",
+     "test/test_fuzzy_indexing.rb",
+     "test/test_index.rb",
      "test/test_lunar.rb",
-     "test/test_lunar_fuzzy.rb",
      "test/test_lunar_fuzzy_word.rb",
-     "test/test_lunar_index.rb",
-     "test/test_lunar_scoring.rb",
-     "test/test_lunar_search.rb",
-     "test/test_sets.rb",
-     "vendor/nest/nest.rb",
-     "vendor/redis/.gitignore",
-     "vendor/redis/LICENSE",
-     "vendor/redis/README.markdown",
-     "vendor/redis/Rakefile",
-     "vendor/redis/benchmarking/logging.rb",
-     "vendor/redis/benchmarking/pipeline.rb",
-     "vendor/redis/benchmarking/speed.rb",
-     "vendor/redis/benchmarking/suite.rb",
-     "vendor/redis/benchmarking/worker.rb",
-     "vendor/redis/bin/distredis",
-     "vendor/redis/examples/basic.rb",
-     "vendor/redis/examples/dist_redis.rb",
-     "vendor/redis/examples/incr-decr.rb",
-     "vendor/redis/examples/list.rb",
-     "vendor/redis/examples/pubsub.rb",
-     "vendor/redis/examples/sets.rb",
-     "vendor/redis/lib/edis.rb",
-     "vendor/redis/lib/redis.rb",
-     "vendor/redis/lib/redis/client.rb",
-     "vendor/redis/lib/redis/dist_redis.rb",
-     "vendor/redis/lib/redis/distributed.rb",
-     "vendor/redis/lib/redis/hash_ring.rb",
-     "vendor/redis/lib/redis/pipeline.rb",
-     "vendor/redis/lib/redis/raketasks.rb",
-     "vendor/redis/lib/redis/subscribe.rb",
-     "vendor/redis/profile.rb",
-     "vendor/redis/tasks/redis.tasks.rb",
-     "vendor/redis/test/db/.gitignore",
-     "vendor/redis/test/distributed_test.rb",
-     "vendor/redis/test/redis_test.rb",
-     "vendor/redis/test/test.conf",
-     "vendor/redis/test/test_helper.rb"
+     "test/test_lunar_nest.rb",
+     "test/test_scoring.rb"
   ]
   s.homepage = %q{http://github.com/sinefunc/lunar}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
-  s.summary = %q{a minimalistic full text search implementation in redis}
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{A redis based full text search engine}
   s.test_files = [
     "test/helper.rb",
+     "test/test_fuzzy_indexing.rb",
+     "test/test_index.rb",
      "test/test_lunar.rb",
-     "test/test_lunar_fuzzy.rb",
      "test/test_lunar_fuzzy_word.rb",
-     "test/test_lunar_index.rb",
-     "test/test_lunar_scoring.rb",
-     "test/test_lunar_search.rb",
-     "test/test_sets.rb",
-     "examples/ohm.rb"
+     "test/test_lunar_nest.rb",
+     "test/test_scoring.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -98,15 +63,21 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<redis>, [">= 2.0.0"])
+      s.add_runtime_dependency(%q<nest>, [">= 0.0.4"])
+      s.add_runtime_dependency(%q<text>, [">= 0"])
       s.add_development_dependency(%q<contest>, [">= 0"])
-      s.add_development_dependency(%q<mocha>, [">= 0"])
     else
+      s.add_dependency(%q<redis>, [">= 2.0.0"])
+      s.add_dependency(%q<nest>, [">= 0.0.4"])
+      s.add_dependency(%q<text>, [">= 0"])
       s.add_dependency(%q<contest>, [">= 0"])
-      s.add_dependency(%q<mocha>, [">= 0"])
     end
   else
+    s.add_dependency(%q<redis>, [">= 2.0.0"])
+    s.add_dependency(%q<nest>, [">= 0.0.4"])
+    s.add_dependency(%q<text>, [">= 0"])
     s.add_dependency(%q<contest>, [">= 0"])
-    s.add_dependency(%q<mocha>, [">= 0"])
   end
 end
 

data/test/helper.rb

@@ -1,14 +1,13 @@
 require 'rubygems'
 require 'test/unit'
 require 'contest'
-require 'mocha'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'lunar'
 
-$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../vendor/redis/lib')
-require 'redis'
-
 class Test::Unit::TestCase
-end
+  def setup
+    Lunar.redis.flushdb
+  end
+end

data/test/test_fuzzy_indexing.rb

@@ -0,0 +1,105 @@
+require 'helper'
+
+class LunarFuzzyTest < Test::Unit::TestCase
+  context "when setting fuzzy name, 'Yukihiro Matsumoto'" do
+    setup do
+      @index = Lunar.index 'Item' do |i|
+        i.id 1001
+        i.fuzzy :name, 'Yukihiro Matsumoto'
+      end
+    end
+
+    should "store Lunar:Item:Fuzzies:name:Y up to o and M up to o" do
+      fname, lname = 'yukihiro', 'matsumoto'
+      nest = Lunar.nest[:Item][:Fuzzies][:name]
+
+      (1..fname.length).each do |length|
+        key = nest[encode(fname[0, length])]
+        assert_equal '1', key.zscore(1001)
+      end
+
+      (1..lname.length).each do |length|
+        key = nest[encode(lname[0, length])]
+        assert_equal '1', key.zscore(1001)
+      end
+
+      assert_equal %w{matsumoto yukihiro},
+        Lunar.nest[:Item][:Fuzzies][1001][:name].smembers.sort
+    end
+  end
+
+  context "when creating an index that already exists" do
+    setup do
+      @index = Lunar.index 'Item' do |i|
+        i.id  1001
+        i.fuzzy :name, 'Yukihiro Matsumoto'
+      end
+
+      @index = Lunar.index 'Item' do |i|
+        i.id  1001
+        i.fuzzy :name, 'Martin Fowler Yuki'
+      end
+    end
+
+    should "remove all fuzzy entries for Yukihiro Matsumoto" do
+      fname, lname = 'yukihiro', 'matsumoto'
+      nest = Lunar.nest[:Item][:Fuzzies][:name]
+
+      (5..fname.length).each do |length|
+        key = nest[encode(fname[0, length])]
+        assert_nil key.zscore('1001')
+      end
+
+      (3..lname.length).each do |length|
+        key = nest[encode(lname[0, length])]
+        assert_nil key.zscore('1001')
+      end
+
+      assert_equal %w{fowler martin yuki},
+        Lunar.nest[:Item][:Fuzzies][1001][:name].smembers.sort
+    end
+
+    should "store Lunar:Item:name:M up to n and F up to r etc..." do
+      fname, lname, triple = 'martin', 'fowler', 'yuki'
+      nest = Lunar.nest[:Item][:Fuzzies][:name]
+
+      %w{martin fowler yuki}.each do |word|
+        (1..word.length).each do |length|
+          key = nest[encode(word[0, length])]
+          assert_equal '1', key.zscore(1001)
+        end
+      end
+    end
+  end
+
+  context "on delete" do
+    setup do
+      @index = Lunar.index 'Item' do |i|
+        i.id 1001
+        i.fuzzy :name, 'Yukihiro Matsumoto'
+      end
+
+      Lunar.delete('Item', 1001)
+    end
+
+    should "remove all fuzzy entries for Yukihiro Matsumoto" do
+      nest = Lunar.nest[:Item][:Fuzzies][:name]
+
+      %w{yukihiro matsumoto}.each do |word|
+        (1..word.length).each do |length|
+          key = nest[encode(word[0, length])]
+          assert_nil key.zscore(1001)
+        end
+      end
+    end
+
+    should "also remove the key Lunar:Item:Fuzzies:1001:name" do
+      assert ! Lunar.redis.exists("Lunar:Item:Fuzzies:1001:name")
+    end
+  end
+
+protected
+  def encode(str)
+    Lunar.encode(str)
+  end
+end