picky 4.25.3 → 4.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c80310289f311bff593cc485ba9a16a528748ac5
-  data.tar.gz: 81cbd3f12b05b4b7c709b1eefede543ba0b18ae2
+  metadata.gz: 361f560a9705c05e6e4105e283edb8af462f6d12
+  data.tar.gz: 6e7430ed38b525ed918cfc8633659ce6f978f142
 SHA512:
-  metadata.gz: 5c1193dd935e6cec656979c20c78bcd0d7913ed1205573add458adaccea6e987842bfe356d6fb6dd500a9bf932e8d760917eff0024f9539eea5afe8026ab7176
-  data.tar.gz: 4aaf6e8987190d760ce4c60f07a7c4d694de5943bda4458176caf5db961e3693abc0c2bda91d879b48ee5625d603ec179b3cdbcb357d7a0c049c6e340468dbaa
+  metadata.gz: d7fd4d1022e3ea493a015579a4020b2d30a6058c0231fe3429e4ca255f697e30e78391afc5591b6f229ad504fed5b3f2b1a35c12e373243a494bb2275c54df80
+  data.tar.gz: fa28a3b92a92ba1d6668ee52e5dd1c9dfc3802b1cd47e2262b092c6469af9c79a1c56fbfda8838f1e545b1ced50a813dc98b48ea130cb0b735254c85f377d3a5

data/lib/picky/category_indexed.rb

@@ -25,7 +25,11 @@ module Picky
           weight && (weight + (sum || 0)) || sum
         end
       else
-        bundle.weight token.text
+        if tokenizer && tokenizer.stemmer?
+          bundle.weight token.stem(tokenizer)
+        else
+          bundle.weight token.text
+        end
       end
     end
 
@@ -45,14 +49,18 @@ module Picky
           ids.empty? ? result : result << ids
         end.flatten
       else
-        bundle.ids token.text
+        # Optimization
+        if tokenizer && tokenizer.stemmer?
+          bundle.ids token.stem(tokenizer)
+        else
+          bundle.ids token.text
+        end
       end
     end
 
     # Returns the right index bundle for this token.
     #
     def bundle_for token
-      # token.partial? ? partial : exact
       token.select_bundle exact, partial
     end
 

data/lib/picky/query/token.rb

@@ -82,6 +82,22 @@ module Picky
       def select_bundle exact, partial
         @partial ? partial : exact
       end
+      
+      # Generates a reused stem.
+      #
+      # TODO Probably should not cache, as not
+      # the same stemmer will be used always.
+      #
+      def stem tokenizer
+        if stem?
+          @stem ||= tokenizer.stem(@text)
+        else
+          @text
+        end
+      end
+      def stem?
+        @text !~ @@no_partial
+      end
 
       # Partial is a conditional setter.
       #

data/lib/picky/tokenizer.rb

@@ -282,7 +282,7 @@ ERROR
     #
     def tokens_for words
       words.collect! { |word| word.downcase!; word } if downcase?
-      words.collect! { |word| stem word } if stemmer?
+      words.collect! { |word| stem word } if stemmer? # Usually only done in indexing step.
       words
     end
 

data/spec/functional/stemming_spec.rb

@@ -6,18 +6,19 @@ require 'stemmer'
 require 'lingua/stemmer'
 
 describe 'stemming' do
-  let(:stemmer) {
-    # Fast stemmer does not conform with the API.
-    #
-    module Stemmer
-      class << self
-        alias_method :stem, :stem_word
-      end
-    end
-    Stemmer
-  }
   
-  describe 'examples' do
+  describe 'per-index stemming' do
+    let(:stemmer) {
+      # Fast stemmer does not conform with the API.
+      #
+      module Stemmer
+        class << self
+          alias_method :stem, :stem_word
+        end
+      end
+      Stemmer
+    }
+    
     it 'works correctly' do
       tokenizer = Picky::Tokenizer.new(stems_with: stemmer)
       
@@ -58,15 +59,7 @@ describe 'stemming' do
 
       try = Picky::Search.new index
       
-      # If you don't stem in the search, it should not be found!
-      #
-      try.search("text:stemming").ids.should == []
-
-      try = Picky::Search.new index do
-        searching stems_with: Stemmer
-      end
-      
-      # With stemming in search AND indexing, it works :)
+      # Stems for both, so finds both.
       #
       try.search("text:stemming").ids.should == [2, 1]
       try.search("text:lem").ids.should == [2]
@@ -80,27 +73,52 @@ describe 'stemming' do
         # eg. Lemming!, then stemming won't work.
         #
         indexing removes_characters: /[^a-z\s]/i,
-                 stems_with: Lingua::Stemmer.new
+                 stems_with: Lingua::Stemmer.new # Both stem
         category :text
       end
-      
+
       index.replace_from id: 1, text: "Hello good Sirs, these things here need stems to work!"
       index.replace_from id: 2, text: "Stemming Lemming!"
 
       try = Picky::Search.new index
-      
-      # If you don't stem in the search, it should not be found!
-      #
-      try.search("text:stemming").ids.should == []
 
-      try = Picky::Search.new index do
-        searching stems_with: Lingua::Stemmer.new
-      end
-      
-      # With stemming in search AND indexing, it works :)
-      #
       try.search("text:stemming").ids.should == [2, 1]
       try.search("text:lem").ids.should == [2]
     end
   end
+  
+  describe 'per-category stemming' do
+    describe 'mixed stemming categories' do
+      it 'stems some but not others' do
+        index = Picky::Index.new :stemming do
+          # Be aware that if !s are not removed from
+          # eg. Lemming!, then stemming won't work.
+          #
+          indexing removes_characters: /[^a-z\s]/i
+          category :text1,
+                   partial: Picky::Partial::None.new,
+                   indexing: { stems_with: Lingua::Stemmer.new }
+          category :text2,
+                   partial: Picky::Partial::None.new
+        end
+    
+        index.replace_from id: 1, text1: 'stemming', text2: 'ios'
+        index.replace_from id: 2, text1: 'ios', text2: 'stemming'
+
+        try = Picky::Search.new index
+    
+        try.search("text1:stemming").ids.should == [1]
+        try.search("text2:ios").ids.should == [1]
+    
+        try.search("text1:ios").ids.should == [2]
+        try.search("text2:stemming").ids.should == [2]
+      
+        try.search("text1:stem").ids.should == [1]
+        try.search("text2:io").ids.should == []
+      
+        try.search("text1:io").ids.should == [2]
+        try.search("text2:stem").ids.should == []
+      end
+    end
+  end
 end

data/spec/functional/symbol_keys_spec.rb

@@ -12,9 +12,7 @@ describe "Option symbol_keys" do
     Picky::Search.new(index) { symbol_keys }
   end
 
-  # Test the enumerator abilities.
-  #
-  it 'can enumerate through the allocations' do
+  it 'returns results' do
     index.category :text
 
     thing = OpenStruct.new id: 1, text: "ohai"
@@ -25,5 +23,45 @@ describe "Option symbol_keys" do
 
     try.search("text:ohai").ids.should == [2, 1]
   end
+  
+  it 'works with facets' do
+    index.category :text
+
+    thing = OpenStruct.new id: 1, text: "ohai"
+    other = OpenStruct.new id: 2, text: "ohai kthxbye"
+
+    index.add thing
+    index.add other
+
+    index.facets(:text).should == { ohai: 2, kthxbye: 1 }
+    try.facets(:text).should == { ohai: 2, kthxbye: 1 }
+  end
+  
+  it 'actually uses symbols - paranoia' do
+    index.category :text
+
+    thing = OpenStruct.new id: 1, text: "ohai"
+    
+    index.add thing
+    
+    index[:text].exact.inverted[:ohai].should == [1]
+    index[:text].exact.weights[:ohai].should == 0.0
+    index[:text].exact.realtime[1].should == [:ohai]
+    index[:text].exact.similarity[:ohai].should == nil
+  end
+  
+  it 'does the internals right - uses symbols' do
+    index.category :text
+
+    thing = OpenStruct.new id: 1, text: "ohai"
+    
+    index.add thing
+    
+    index[:text].exact.inverted.should == { ohai: [1] }
+    index[:text].exact.weights.should == { ohai: 0.0 }
+    # TODO This could be removed if sorting was always explicitly done.
+    index[:text].exact.realtime.should == { 1 => [:ohai] }
+    index[:text].exact.similarity.should == {}
+  end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: picky
 version: !ruby/object:Gem::Version
-  version: 4.25.3
+  version: 4.26.0
 platform: ruby
 authors:
 - Florian Hanke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-20 00:00:00.000000000 Z
+date: 2015-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport