picky 4.13.1 → 4.14.0

data/lib/picky/category.rb

@@ -28,6 +28,7 @@ module Picky
     #
     # Advanced Options:
     #  * source: Use if the category should use a different source.
+    #  * tokenize: Whether to use the tokenizer (default is true).
     #  * tokenizer: Use a subclass of Tokenizers::Base that implements #tokens_for and #empty_tokens.
     #  * weight: Weights::Logarithmic.new, Weights::Constant.new(int = 0),
     #  Weights::Dynamic.new(&block) or an object that responds
@@ -47,6 +48,7 @@ module Picky
       # Instantly extracted to raise an error instantly.
       #
       @source    = Source.from options[:source], true, @index.name
+      @tokenize  = options[:tokenize] != false
       @tokenizer = Tokenizer.from options[:indexing], @index.name, name
       @ranger    = options[:ranging] || Range
 
@@ -78,7 +80,7 @@ module Picky
     #
     # TODO Rewrite it such that this does not need to be maintained separately.
     #
-    @@known_keys = [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :weight]
+    @@known_keys = [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :tokenize, :tokenizer, :weight]
     def warn_if_unknown options
       warn <<-WARNING if options && (options.keys - @@known_keys).size > 0
 

data/lib/picky/category_indexing.rb

@@ -95,8 +95,10 @@ module Picky
     # If one isn't set on this category, will try the index,
     # and finally the default index tokenizer.
     #
+    # Will return nil if tokenize is set to false.
+    #
     def tokenizer
-      @tokenizer || @index.tokenizer
+      @tokenizer || @index.tokenizer if @tokenize
     end
 
     # Clears the caches.

data/lib/picky/category_realtime.rb

@@ -57,10 +57,17 @@ module Picky
     # For the given id, adds the list of
     # strings to the index for the given id.
     #
-    def add_text id, text, where = :unshift
+    def add_text id, text_or_tokens, where = :unshift
       # text = text.to_sym if @symbols # SYMBOLS.
-      tokens, _ = tokenizer.tokenize text
+      tokens = nil
+      if tokenizer
+        tokens, _ = tokenizer.tokenize text_or_tokens
+      else
+        tokens = text_or_tokens
+      end
       tokens.each { |text| add_tokenized_token id.send(key_format), text, where, false }
+    rescue NoMethodError
+      raise %Q{You probably set tokenize: false on category "#{name}". It will need an Enumerator of previously tokenized tokens.}
     end
 
     #

data/lib/picky/indexers/parallel.rb

@@ -58,12 +58,26 @@ module Picky
       def index_flush objects, file, category, cache, tokenizer
         comma   = ?,
         newline = ?\n
-
-        objects.each do |object|
-          tokens, _ = tokenizer.tokenize object.send(category.from) # Note: Originals not needed.
-          tokens.each do |token_text|
-            next unless token_text
-            cache << object.id << comma << token_text << newline
+        
+        # Optimized, therefore duplicate code.
+        #
+        # TODO Deoptimize?
+        #
+        if tokenizer
+          objects.each do |object|
+            tokens, _ = tokenizer.tokenize object.send(category.from) # Note: Originals not needed.
+            tokens.each do |token_text|
+              next unless token_text
+              cache << object.id << comma << token_text << newline
+            end
+          end
+        else
+          objects.each do |object|
+            tokens = object.send(category.from) # Note: Originals not needed.
+            tokens.each do |token_text|
+              next unless token_text
+              cache << object.id << comma << token_text << newline
+            end
           end
         end
 

data/lib/picky/indexers/serial.rb

@@ -54,12 +54,25 @@ module Picky
       def index_flush datas, file, cache, tokenizer
         comma   = ?,
         newline = ?\n
-
-        datas.each do |indexed_id, text|
-          tokens, _ = tokenizer.tokenize text # Note: Originals not needed.
-          tokens.each do |token_text|
-            next unless token_text
-            cache << indexed_id << comma << token_text << newline
+        
+        # Optimized, therefore duplicate code.
+        #
+        # TODO Deoptimize?
+        #
+        if tokenizer
+          datas.each do |indexed_id, text|
+            tokens, _ = tokenizer.tokenize text # Note: Originals not needed.
+            tokens.each do |token_text|
+              next unless token_text
+              cache << indexed_id << comma << token_text << newline
+            end
+          end
+        else
+          datas.each do |indexed_id, tokens|
+            tokens.each do |token_text|
+              next unless token_text
+              cache << indexed_id << comma << token_text << newline
+            end
           end
         end
 

data/spec/functional/backends/no_tokenize_spec.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe 'Category#tokenize(false)' do
+  
+  it 'does tokenize' do
+    index = Picky::Index.new :thing do
+      category :text, tokenize: true
+    end
+
+    thing = Struct.new :id, :text
+    # expect do # Does not fail – because #to_s is called on the Array.
+    index.add thing.new(1, ['already', 'tokenized'])
+    # end.to raise_error
+    index.add thing.new(2, 'this should fail')
+    
+    try = Picky::Search.new index
+    
+    try.search('already').ids.should == [] # Not found because ['already', is indexed.
+  end
+  it 'does not tokenize' do
+    index = Picky::Index.new :thing do
+      category :text, tokenize: false
+    end
+
+    thing = Struct.new :id, :text
+    index.add thing.new(1, ['already', 'tokenized'])
+    expect do
+      index.add thing.new(2, 'this should fail')
+    end.to raise_error('You probably set tokenize: false on category "text". It will need an Enumerator of previously tokenized tokens.')
+    
+    try = Picky::Search.new index
+    
+    try.search('already').ids.should == [1]
+  end
+  
+end

data/spec/lib/category_spec.rb

@@ -52,7 +52,7 @@ describe Picky::Category do
       category.should_receive(:warn).once.with <<-WARNING
 
 Warning: Category options {:weights=>:some_weight} for category some_category contain an unknown option.
-         Working options are: [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :weight].
+         Working options are: [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :tokenize, :tokenizer, :weight].
 WARNING
       
       category.warn_if_unknown :weights => :some_weight

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: picky
 version: !ruby/object:Gem::Version
-  version: 4.13.1
+  version: 4.14.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-01 00:00:00.000000000 Z
+date: 2013-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70248447814120 !ruby/object:Gem::Requirement
+  requirement: &70147105922620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,21 +21,21 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70248447814120
+  version_requirements: *70147105922620
 - !ruby/object:Gem::Dependency
   name: picky-client
-  requirement: &70248447750680 !ruby/object:Gem::Requirement
+  requirement: &70147105922020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 4.13.1
+        version: 4.14.0
   type: :development
   prerelease: false
-  version_requirements: *70248447750680
+  version_requirements: *70147105922020
 - !ruby/object:Gem::Dependency
   name: text
-  requirement: &70248447748820 !ruby/object:Gem::Requirement
+  requirement: &70147105921540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70248447748820
+  version_requirements: *70147105921540
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70248447747100 !ruby/object:Gem::Requirement
+  requirement: &70147105921060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70248447747100
+  version_requirements: *70147105921060
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70248447744720 !ruby/object:Gem::Requirement
+  requirement: &70147105920460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70248447744720
+  version_requirements: *70147105920460
 - !ruby/object:Gem::Dependency
   name: rack_fast_escape
-  requirement: &70248447728420 !ruby/object:Gem::Requirement
+  requirement: &70147105919900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70248447728420
+  version_requirements: *70147105919900
 description: Fast Ruby semantic text search engine with comfortable single field interface.
 email: florian.hanke+picky@gmail.com
 executables:
@@ -239,6 +239,7 @@ files:
 - spec/functional/backends/memory_bundle_realtime_spec.rb
 - spec/functional/backends/memory_json_utf8_spec.rb
 - spec/functional/backends/memory_spec.rb
+- spec/functional/backends/no_tokenize_spec.rb
 - spec/functional/backends/redis_bundle_realtime_spec.rb
 - spec/functional/backends/redis_spec.rb
 - spec/functional/backends/special_spec.rb
@@ -407,6 +408,7 @@ test_files:
 - spec/functional/backends/memory_bundle_realtime_spec.rb
 - spec/functional/backends/memory_json_utf8_spec.rb
 - spec/functional/backends/memory_spec.rb
+- spec/functional/backends/no_tokenize_spec.rb
 - spec/functional/backends/redis_bundle_realtime_spec.rb
 - spec/functional/backends/redis_spec.rb
 - spec/functional/backends/special_spec.rb