picky 3.4.0 → 3.4.1

data/lib/picky/backends/memory/json.rb

@@ -17,7 +17,7 @@ module Picky
         # Loads the index hash from json format.
         #
         def load
-          Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true
+          Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true # TODO to_sym
         end
 
         # Dumps the index hash in json format.

data/lib/picky/backends/redis.rb

@@ -46,6 +46,40 @@ module Picky
         String.new(client, "#{bundle.identifier}:configuration")
       end
 
+      # Does the Redis version already include
+      # scripting support?
+      #
+      def redis_with_scripting?
+        at_least_version redis_version, [2, 6, 0]
+      end
+
+      # Compares two versions each in an array [major, minor, patch]
+      # format and returns true if the first version is higher
+      # or the same as the second one. False if not.
+      #
+      # Note: Destructive.
+      #
+      def at_least_version major_minor_patch, should_be
+        3.times { return false if major_minor_patch.shift < should_be.shift }
+        true
+      end
+
+      # Returns an array describing the
+      # current Redis version.
+      #
+      # Note: This method assumes that clients answer
+      #       to #info with a hash (string/symbol keys)
+      #       detailing the infos.
+      #
+      # Example:
+      #   backend.redis_version # => [2, 4, 1]
+      #
+      def redis_version
+        infos          = client.info
+        version_string = infos['redis_version'] || infos[:redis_version]
+        version_string.split('.').map &:to_i
+      end
+
       # Returns the result ids for the allocation.
       #
       # Developers wanting to program fast intersection
@@ -54,29 +88,53 @@ module Picky
       #
       # Note: We use the amount and offset hints to speed Redis up.
       #
-      def ids combinations, amount, offset
-        identifiers = combinations.inject([]) do |identifiers, combination|
-          identifiers << "#{combination.identifier}"
-        end
-
-        result_id = generate_intermediate_result_id
-
-        # Intersect and store.
-        #
-        client.zinterstore result_id, identifiers
-
-        # Get the stored result.
-        #
-        results = client.zrange result_id, offset, (offset + amount)
-
-        # Delete the stored result as it was only for temporary purposes.
-        #
-        # Note: I could also not delete it, but that would not be clean at all.
-        #
-        client.del result_id
-
-        results
-      end
+#       def ids combinations, amount, offset
+#         if redis_with_scripting?
+#           @@script = <<-SCRIPT
+# redis.call('zinterstore', KEYS[1], ARGV[1]);
+# local result = redis.call('zrange', KEYS[1], ARGV[2], ARGV[3])
+# redis.call('del', KEYS[1])
+# return result
+# SCRIPT
+#           # Scripting version of #ids.
+#           #
+#           def ids combinations, amount, offset
+#             identifiers = combinations.inject([]) do |identifiers, combination|
+#               identifiers << "#{combination.identifier}"
+#             end
+#
+#             # Assume it's using EVALSHA.
+#             #
+#             client.eval @@script, generate_intermediate_result_id, identifiers, offset, (offset + amount)
+#           end
+#         else
+          # Non-Scripting version of #ids.
+          #
+          def ids combinations, amount, offset
+            identifiers = combinations.inject([]) do |identifiers, combination|
+              identifiers << "#{combination.identifier}"
+            end
+
+            result_id = generate_intermediate_result_id
+
+            # Intersect and store.
+            #
+            client.zinterstore result_id, identifiers
+
+            # Get the stored result.
+            #
+            results = client.zrange result_id, offset, (offset + amount)
+
+            # Delete the stored result as it was only for temporary purposes.
+            #
+            # Note: I could also not delete it, but that would not be clean at all.
+            #
+            client.del result_id
+
+            results
+          end
+        # end
+      # end
 
       # Generate a multiple host/process safe result id.
       #

data/lib/picky/extensions/string.rb

@@ -0,0 +1,45 @@
+# Extending the String class.
+#
+class String # :nodoc:all
+
+  # 'keys'.each_subtoken    # => yields each of ['keys', 'key', 'ke', 'k']
+  # 'keys'.each_subtoken(2) # => yields each of ['keys', 'key', 'ke']
+  #
+  def each_subtoken from_length = 1, range = nil
+    sub = self
+
+    sub = sub[range] if range
+
+    yield sub
+
+    size = sub.size
+    from_length = size + from_length + 1 if from_length < 0
+    from_length = size if size < from_length
+    from_length = 1 if from_length < 1
+
+    size.downto(from_length + 1) { yield sub = sub.chop }
+  end
+
+  # 'keys'.each_intoken         # => yields each of ['keys', 'key', 'eys', 'ke', 'ey', 'ys', 'k', 'e', 'y', 's']
+  # 'keys'.each_intoken(2)      # => yields each of ['keys', 'key', 'eys', 'ke', 'ey', 'ys']
+  # 'keys'.each_intoken(2, 3)   # => yields each of ['key', 'eys', 'ke', 'ey', 'ys']
+  # 'keys'.each_intoken(10, 12) # => yields nothing (min larger than str)
+  #
+  def each_intoken min_length = 1, max_length = -1
+    max_length = size + max_length + 1 if max_length < 0
+    max_length = size if size < max_length
+    max_length = 1 if max_length < 1
+
+    min_length = size + min_length + 1 if min_length < 0
+    min_length = 1 if min_length < 1
+
+    this_many = size - max_length + 1
+    max_length.downto(min_length) do |length|
+      this_many.times do |offset|
+        yield self[offset, length]
+      end
+      this_many += 1
+    end
+  end
+
+end

data/lib/picky/extensions/symbol.rb

@@ -5,15 +5,18 @@ class Symbol # :nodoc:all
   # :keys.each_subtoken    # => yields each of [:keys, :key, :ke, :k]
   # :keys.each_subtoken(2) # => yields each of [:keys, :key, :ke]
   #
-  def each_subtoken from_length = 1
+  def each_subtoken from_length = 1, range = nil
     sub = self.id2name
 
+    sub = sub[range] if range
+
+    yield sub.intern
+
     size = sub.size
     from_length = size + from_length + 1 if from_length < 0
     from_length = size if size < from_length
     from_length = 1 if from_length < 1
 
-    yield self
     size.downto(from_length + 1) { yield sub.chop!.intern }
   end
 

data/lib/picky/generators/partial/substring.rb

@@ -20,11 +20,13 @@ module Picky
           else
             if @from < 0 && @to < 0
               def each_subtoken token, &block
-                token[0..@to].intern.each_subtoken @from - @to - 1, &block
+                token.each_subtoken @from - @to - 1, (0..@to), &block
+                # token[0..@to].intern.each_subtoken @from - @to - 1, &block # TODO to_sym
               end
             else
               def each_subtoken token, &block
-                token[0..@to].intern.each_subtoken @from, &block
+                token.each_subtoken @from, (0..@to), &block
+                # token[0..@to].intern.each_subtoken @from, &block # TODO to_sym
               end
             end
           end

data/lib/picky/loader.rb

@@ -56,6 +56,7 @@ module Picky
       load_relative 'extensions/object'
       load_relative 'extensions/array'
       load_relative 'extensions/symbol'
+      load_relative 'extensions/string'
       load_relative 'extensions/module'
       load_relative 'extensions/class'
       load_relative 'extensions/hash'

data/lib/picky/query/token.rb

@@ -37,7 +37,7 @@ module Picky
         partialize # TODO Should this operate on the original?
         similarize # TODO Should this operate on the original?
         remove_illegals # TODO Remove?
-        symbolize
+        symbolize # TODO to_sym
         self
       end
 

data/spec/lib/backends/redis_spec.rb

@@ -12,6 +12,32 @@ describe Picky::Backends::Redis do
       @backend.stub! :timed_exclaim
     end
 
+    describe "redis_with_scripting?" do
+      let(:client) { stub :client}
+      let(:redis) { described_class.new client: client }
+
+      it "answers correctly" do
+        client.stub! :info => {"redis_version"=>"2.2.2", "redis_git_sha1"=>"00000000", "redis_git_dirty"=>"0", "arch_bits"=>"64", "multiplexing_api"=>"kqueue", "process_id"=>"70364", "uptime_in_seconds"=>"86804", "uptime_in_days"=>"1", "lru_clock"=>"2057342", "used_cpu_sys"=>"12.76", "used_cpu_user"=>"13.44", "used_cpu_sys_childrens"=>"0.62", "used_cpu_user_childrens"=>"0.30", "connected_clients"=>"1", "connected_slaves"=>"0", "client_longest_output_list"=>"0", "client_biggest_input_buf"=>"0", "blocked_clients"=>"0", "used_memory"=>"34389632", "used_memory_human"=>"32.80M", "used_memory_rss"=>"675840", "mem_fragmentation_ratio"=>"0.02", "use_tcmalloc"=>"0", "loading"=>"0", "aof_enabled"=>"0", "changes_since_last_save"=>"0", "bgsave_in_progress"=>"0", "last_save_time"=>"1320721195", "bgrewriteaof_in_progress"=>"0", "total_connections_received"=>"20", "total_commands_processed"=>"327594", "expired_keys"=>"0", "evicted_keys"=>"0", "keyspace_hits"=>"218584", "keyspace_misses"=>"98664", "hash_max_zipmap_entries"=>"512", "hash_max_zipmap_value"=>"64", "pubsub_channels"=>"0", "pubsub_patterns"=>"0", "vm_enabled"=>"0", "role"=>"master", "allocation_stats"=>"2=74,6=1,8=15,9=156,10=142939,11=104891,12=290902,13=263791,14=14570,15=29143,16=1661979,17=8986,18=6370,19=4508,20=18010,21=1622,22=1136,23=544,24=419271,25=154,26=73,27=32,28=30,29=14,32=419046,33=6,34=7,35=15,36=10,37=12,38=625,39=7127,40=207716,41=40840,42=7246,43=2645,44=28390,45=37835,46=35164,47=67465,48=54765,49=41247,50=44391,51=36420,52=29582,53=21491,54=18575,55=14101,56=61954,57=5476,58=3246,59=2227,60=1502,61=868,62=541,63=282,64=69006,65=87,66=58,67=32,68=30,69=6,70=2,71=5,72=12723,74=19,75=2,76=13,77=6,80=12500,81=10,82=4,83=8,84=10,85=14,86=5,87=37,88=97714,89=58,91=30,93=68,94=14,95=35,97=56,99=46,101=24,103=17,104=846,105=1,106=15,107=19,109=4,110=6,111=13,113=8,114=8,115=4,116=5,117=8,118=11,119=4,120=217,121=18,122=6,125=5,126=12,128=4411,131=12,133=8,136=57,137=14,138=10,142=6,143=4,145=6,147=14,150=4,152=23,153=6,157=4,158=6,159=14,163=8,164=18,166=4,168=6,169=1,170=16,171=27,173=7,174=10,175=31,177=14,178=6,179=13,181=39,182=4,183=12,184=7,185=42,187=16,189=69,191=22,193=17,195=8,196=16,197=23,199=20,201=23,203=12,205=4,206=16,207=6,208=4,209=10,211=1,213=8,215=4,216=10,217=4,218=14,219=10,221=14,223=4,225=8,226=6,227=4,228=10,230=10,232=6,234=6,237=4,238=6,239=8,240=6,241=4,242=6,245=6,248=4,249=16,250=6,252=4,253=4,>=256=113463", "db15"=>"keys=26605,expires=0"}
+
+        redis.redis_with_scripting?.should == false
+      end
+      it "answers correctly" do
+        client.stub! :info => {"redis_version"=>"2.6.0", "redis_git_sha1"=>"00000000", "redis_git_dirty"=>"0", "arch_bits"=>"64", "multiplexing_api"=>"kqueue", "process_id"=>"70364", "uptime_in_seconds"=>"86804", "uptime_in_days"=>"1", "lru_clock"=>"2057342", "used_cpu_sys"=>"12.76", "used_cpu_user"=>"13.44", "used_cpu_sys_childrens"=>"0.62", "used_cpu_user_childrens"=>"0.30", "connected_clients"=>"1", "connected_slaves"=>"0", "client_longest_output_list"=>"0", "client_biggest_input_buf"=>"0", "blocked_clients"=>"0", "used_memory"=>"34389632", "used_memory_human"=>"32.80M", "used_memory_rss"=>"675840", "mem_fragmentation_ratio"=>"0.02", "use_tcmalloc"=>"0", "loading"=>"0", "aof_enabled"=>"0", "changes_since_last_save"=>"0", "bgsave_in_progress"=>"0", "last_save_time"=>"1320721195", "bgrewriteaof_in_progress"=>"0", "total_connections_received"=>"20", "total_commands_processed"=>"327594", "expired_keys"=>"0", "evicted_keys"=>"0", "keyspace_hits"=>"218584", "keyspace_misses"=>"98664", "hash_max_zipmap_entries"=>"512", "hash_max_zipmap_value"=>"64", "pubsub_channels"=>"0", "pubsub_patterns"=>"0", "vm_enabled"=>"0", "role"=>"master", "allocation_stats"=>"2=74,6=1,8=15,9=156,10=142939,11=104891,12=290902,13=263791,14=14570,15=29143,16=1661979,17=8986,18=6370,19=4508,20=18010,21=1622,22=1136,23=544,24=419271,25=154,26=73,27=32,28=30,29=14,32=419046,33=6,34=7,35=15,36=10,37=12,38=625,39=7127,40=207716,41=40840,42=7246,43=2645,44=28390,45=37835,46=35164,47=67465,48=54765,49=41247,50=44391,51=36420,52=29582,53=21491,54=18575,55=14101,56=61954,57=5476,58=3246,59=2227,60=1502,61=868,62=541,63=282,64=69006,65=87,66=58,67=32,68=30,69=6,70=2,71=5,72=12723,74=19,75=2,76=13,77=6,80=12500,81=10,82=4,83=8,84=10,85=14,86=5,87=37,88=97714,89=58,91=30,93=68,94=14,95=35,97=56,99=46,101=24,103=17,104=846,105=1,106=15,107=19,109=4,110=6,111=13,113=8,114=8,115=4,116=5,117=8,118=11,119=4,120=217,121=18,122=6,125=5,126=12,128=4411,131=12,133=8,136=57,137=14,138=10,142=6,143=4,145=6,147=14,150=4,152=23,153=6,157=4,158=6,159=14,163=8,164=18,166=4,168=6,169=1,170=16,171=27,173=7,174=10,175=31,177=14,178=6,179=13,181=39,182=4,183=12,184=7,185=42,187=16,189=69,191=22,193=17,195=8,196=16,197=23,199=20,201=23,203=12,205=4,206=16,207=6,208=4,209=10,211=1,213=8,215=4,216=10,217=4,218=14,219=10,221=14,223=4,225=8,226=6,227=4,228=10,230=10,232=6,234=6,237=4,238=6,239=8,240=6,241=4,242=6,245=6,248=4,249=16,250=6,252=4,253=4,>=256=113463", "db15"=>"keys=26605,expires=0"}
+
+        redis.redis_with_scripting?.should == true
+      end
+      it "answers correctly" do
+        client.stub! :info => {:redis_version=>"2.2.2", "redis_git_sha1"=>"00000000", "redis_git_dirty"=>"0", "arch_bits"=>"64", "multiplexing_api"=>"kqueue", "process_id"=>"70364", "uptime_in_seconds"=>"86804", "uptime_in_days"=>"1", "lru_clock"=>"2057342", "used_cpu_sys"=>"12.76", "used_cpu_user"=>"13.44", "used_cpu_sys_childrens"=>"0.62", "used_cpu_user_childrens"=>"0.30", "connected_clients"=>"1", "connected_slaves"=>"0", "client_longest_output_list"=>"0", "client_biggest_input_buf"=>"0", "blocked_clients"=>"0", "used_memory"=>"34389632", "used_memory_human"=>"32.80M", "used_memory_rss"=>"675840", "mem_fragmentation_ratio"=>"0.02", "use_tcmalloc"=>"0", "loading"=>"0", "aof_enabled"=>"0", "changes_since_last_save"=>"0", "bgsave_in_progress"=>"0", "last_save_time"=>"1320721195", "bgrewriteaof_in_progress"=>"0", "total_connections_received"=>"20", "total_commands_processed"=>"327594", "expired_keys"=>"0", "evicted_keys"=>"0", "keyspace_hits"=>"218584", "keyspace_misses"=>"98664", "hash_max_zipmap_entries"=>"512", "hash_max_zipmap_value"=>"64", "pubsub_channels"=>"0", "pubsub_patterns"=>"0", "vm_enabled"=>"0", "role"=>"master", "allocation_stats"=>"2=74,6=1,8=15,9=156,10=142939,11=104891,12=290902,13=263791,14=14570,15=29143,16=1661979,17=8986,18=6370,19=4508,20=18010,21=1622,22=1136,23=544,24=419271,25=154,26=73,27=32,28=30,29=14,32=419046,33=6,34=7,35=15,36=10,37=12,38=625,39=7127,40=207716,41=40840,42=7246,43=2645,44=28390,45=37835,46=35164,47=67465,48=54765,49=41247,50=44391,51=36420,52=29582,53=21491,54=18575,55=14101,56=61954,57=5476,58=3246,59=2227,60=1502,61=868,62=541,63=282,64=69006,65=87,66=58,67=32,68=30,69=6,70=2,71=5,72=12723,74=19,75=2,76=13,77=6,80=12500,81=10,82=4,83=8,84=10,85=14,86=5,87=37,88=97714,89=58,91=30,93=68,94=14,95=35,97=56,99=46,101=24,103=17,104=846,105=1,106=15,107=19,109=4,110=6,111=13,113=8,114=8,115=4,116=5,117=8,118=11,119=4,120=217,121=18,122=6,125=5,126=12,128=4411,131=12,133=8,136=57,137=14,138=10,142=6,143=4,145=6,147=14,150=4,152=23,153=6,157=4,158=6,159=14,163=8,164=18,166=4,168=6,169=1,170=16,171=27,173=7,174=10,175=31,177=14,178=6,179=13,181=39,182=4,183=12,184=7,185=42,187=16,189=69,191=22,193=17,195=8,196=16,197=23,199=20,201=23,203=12,205=4,206=16,207=6,208=4,209=10,211=1,213=8,215=4,216=10,217=4,218=14,219=10,221=14,223=4,225=8,226=6,227=4,228=10,230=10,232=6,234=6,237=4,238=6,239=8,240=6,241=4,242=6,245=6,248=4,249=16,250=6,252=4,253=4,>=256=113463", "db15"=>"keys=26605,expires=0"}
+
+        redis.redis_with_scripting?.should == false
+      end
+      it "answers correctly" do
+        client.stub! :info => {:redis_version=>"2.6.0", "redis_git_sha1"=>"00000000", "redis_git_dirty"=>"0", "arch_bits"=>"64", "multiplexing_api"=>"kqueue", "process_id"=>"70364", "uptime_in_seconds"=>"86804", "uptime_in_days"=>"1", "lru_clock"=>"2057342", "used_cpu_sys"=>"12.76", "used_cpu_user"=>"13.44", "used_cpu_sys_childrens"=>"0.62", "used_cpu_user_childrens"=>"0.30", "connected_clients"=>"1", "connected_slaves"=>"0", "client_longest_output_list"=>"0", "client_biggest_input_buf"=>"0", "blocked_clients"=>"0", "used_memory"=>"34389632", "used_memory_human"=>"32.80M", "used_memory_rss"=>"675840", "mem_fragmentation_ratio"=>"0.02", "use_tcmalloc"=>"0", "loading"=>"0", "aof_enabled"=>"0", "changes_since_last_save"=>"0", "bgsave_in_progress"=>"0", "last_save_time"=>"1320721195", "bgrewriteaof_in_progress"=>"0", "total_connections_received"=>"20", "total_commands_processed"=>"327594", "expired_keys"=>"0", "evicted_keys"=>"0", "keyspace_hits"=>"218584", "keyspace_misses"=>"98664", "hash_max_zipmap_entries"=>"512", "hash_max_zipmap_value"=>"64", "pubsub_channels"=>"0", "pubsub_patterns"=>"0", "vm_enabled"=>"0", "role"=>"master", "allocation_stats"=>"2=74,6=1,8=15,9=156,10=142939,11=104891,12=290902,13=263791,14=14570,15=29143,16=1661979,17=8986,18=6370,19=4508,20=18010,21=1622,22=1136,23=544,24=419271,25=154,26=73,27=32,28=30,29=14,32=419046,33=6,34=7,35=15,36=10,37=12,38=625,39=7127,40=207716,41=40840,42=7246,43=2645,44=28390,45=37835,46=35164,47=67465,48=54765,49=41247,50=44391,51=36420,52=29582,53=21491,54=18575,55=14101,56=61954,57=5476,58=3246,59=2227,60=1502,61=868,62=541,63=282,64=69006,65=87,66=58,67=32,68=30,69=6,70=2,71=5,72=12723,74=19,75=2,76=13,77=6,80=12500,81=10,82=4,83=8,84=10,85=14,86=5,87=37,88=97714,89=58,91=30,93=68,94=14,95=35,97=56,99=46,101=24,103=17,104=846,105=1,106=15,107=19,109=4,110=6,111=13,113=8,114=8,115=4,116=5,117=8,118=11,119=4,120=217,121=18,122=6,125=5,126=12,128=4411,131=12,133=8,136=57,137=14,138=10,142=6,143=4,145=6,147=14,150=4,152=23,153=6,157=4,158=6,159=14,163=8,164=18,166=4,168=6,169=1,170=16,171=27,173=7,174=10,175=31,177=14,178=6,179=13,181=39,182=4,183=12,184=7,185=42,187=16,189=69,191=22,193=17,195=8,196=16,197=23,199=20,201=23,203=12,205=4,206=16,207=6,208=4,209=10,211=1,213=8,215=4,216=10,217=4,218=14,219=10,221=14,223=4,225=8,226=6,227=4,228=10,230=10,232=6,234=6,237=4,238=6,239=8,240=6,241=4,242=6,245=6,248=4,249=16,250=6,252=4,253=4,>=256=113463", "db15"=>"keys=26605,expires=0"}
+
+        redis.redis_with_scripting?.should == true
+      end
+    end
+
     describe 'create_...' do
       [
         [:inverted,      Picky::Backends::Redis::Float],

data/spec/lib/extensions/string_spec.rb

@@ -0,0 +1,492 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe String do
+
+  context 'performance' do
+    include Picky::Helpers::Measuring
+    before(:each) do
+      @token = (((0..9).to_a)*10).to_s.to_sym
+    end
+    it "is fast" do
+      performance_of { @token.each_subtoken { |subtoken| } }.should < 0.00065
+    end
+    it 'is fast enough' do
+      performance_of { @token.each_intoken { |intoken| } }.should < 0.025 # TODO Slow!
+    end
+  end
+
+  describe 'each_intoken' do
+    context 'normal String' do
+      before(:each) do
+        @str = "picky"
+      end
+      context 'no params' do
+        it "yields the right elements" do
+          result = []
+          @str.each_intoken do |subtoken|
+            result << subtoken
+          end
+          result.should == ['picky', 'pick', 'icky', 'pic', 'ick', 'cky', 'pi', 'ic', 'ck', 'ky', 'p', 'i', 'c', 'k', 'y']
+        end
+      end
+      context 'with min_length == 0' do
+        it "yields the right elements" do
+          result = []
+          @str.each_intoken(0) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['picky', 'pick', 'icky', 'pic', 'ick', 'cky', 'pi', 'ic', 'ck', 'ky', 'p', 'i', 'c', 'k', 'y']
+        end
+        context 'max_length == 0' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(0, 0) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['p', 'i', 'c', 'k', 'y']
+          end
+        end
+        context 'max_length == 1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(0, 1) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['p', 'i', 'c', 'k', 'y']
+          end
+        end
+        context 'max_length == 2' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(0, 2) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['pi', 'ic', 'ck', 'ky', 'p', 'i', 'c', 'k', 'y']
+          end
+        end
+        context 'max_length == 10' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(0, 10) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['picky', 'pick', 'icky', 'pic', 'ick', 'cky', 'pi', 'ic', 'ck', 'ky', 'p', 'i', 'c', 'k', 'y']
+          end
+        end
+        context 'max_length == -1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(0, -1) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['picky', 'pick', 'icky', 'pic', 'ick', 'cky', 'pi', 'ic', 'ck', 'ky', 'p', 'i', 'c', 'k', 'y']
+          end
+        end
+      end
+      context 'with min_length == sym.size' do
+        it "yields the right elements" do
+          result = []
+          @str.each_intoken(@str.size) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['picky']
+        end
+        context 'max_length == 0' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size, 0) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size, 1) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 2' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size, 2) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 10' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size, 10) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['picky']
+          end
+        end
+        context 'max_length == -1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size, -1) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['picky']
+          end
+        end
+      end
+      context 'with min_length > sym.size' do
+        it "yields the right elements" do
+          result = []
+          @str.each_intoken(@str.size+1) do |subtoken|
+            result << subtoken
+          end
+          result.should == []
+        end
+        context 'max_length == 0' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size+1, 0) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size+1, 1) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 2' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size+1, 2) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 10' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size+1, 10) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == -1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(@str.size+1, -1) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+      end
+      context 'with min_length < 0' do
+        it "yields the right elements" do
+          result = []
+          @str.each_intoken(-2) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['picky', 'pick', 'icky']
+        end
+        context 'max_length == 0' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(-2, 0) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(-2, 1) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 2' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(-2, 2) do |subtoken|
+              result << subtoken
+            end
+            result.should == []
+          end
+        end
+        context 'max_length == 10' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(-2, 10) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['picky', 'pick', 'icky']
+          end
+        end
+        context 'max_length == -1' do
+          it 'yields the right elements' do
+            result = []
+            @str.each_intoken(-2, -1) do |subtoken|
+              result << subtoken
+            end
+            result.should == ['picky', 'pick', 'icky']
+          end
+        end
+      end
+    end
+  end
+
+  describe "each_subtoken" do
+    context 'normal String' do
+      before(:each) do
+        @str = 'reinke'
+      end
+      context 'no downto' do
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke', 'reink', 'rein', 'rei', 're', 'r']
+        end
+      end
+      context 'downto is larger than the String' do
+        before(:each) do
+          @downto = 8
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke']
+        end
+      end
+      context 'downto is exactly the same as String' do
+        before(:each) do
+          @downto = 6
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke']
+        end
+      end
+      context 'downto is smaller than the length of the String' do
+        before(:each) do
+          @downto = 4
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke', 'reink', 'rein']
+        end
+      end
+      context 'downto is 1' do
+        before(:each) do
+          @downto = 1
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke', 'reink', 'rein', 'rei', 're', 'r']
+        end
+      end
+      context 'downto is 0' do
+        before(:each) do
+          @downto = 0
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke', 'reink', 'rein', 'rei', 're', 'r']
+        end
+      end
+      context 'downto is less than zero' do
+        before(:each) do
+          @downto = -2
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['reinke', 'reink']
+        end
+      end
+    end
+    context 'japanese String' do
+      before(:each) do
+        @str = :日本語
+      end
+      it "should return an array of japanese Strings, each 1 smaller than the other" do
+        result = []
+        @str.each_subtoken do |subtoken|
+          result << subtoken
+        end
+        result.should == [:日本語, :日本, :日]
+      end
+    end
+    context 'very short String' do
+      before(:each) do
+        @str = 'r'
+      end
+      context 'no downto' do
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is larger than the String' do
+        before(:each) do
+          @downto = 8
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is exactly the same as String' do
+        before(:each) do
+          @downto = 6
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is smaller than the length of the String' do
+        before(:each) do
+          @downto = 4
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is 1' do
+        before(:each) do
+          @downto = 1
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is 0' do
+        before(:each) do
+          @downto = 0
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is less than zero' do
+        before(:each) do
+          @downto = -1
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is less than zero' do
+        before(:each) do
+          @downto = -2
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is less than zero' do
+        before(:each) do
+          @downto = -3
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+      context 'downto is less than zero' do
+        before(:each) do
+          @downto = -100
+        end
+        it "should return an array of pieces of the original token, each 1 smaller than the other" do
+          result = []
+          @str.each_subtoken(@downto) do |subtoken|
+            result << subtoken
+          end
+          result.should == ['r']
+        end
+      end
+    end
+    context "with range param" do
+      it "returns a subtoken array from a clipped original" do
+        result = []
+        'reinke'.each_subtoken 2, (0..3) do |subtoken|
+          result << subtoken
+        end
+        result.should == ['rein', 'rei', 're']
+      end
+      it "returns a subtoken array from a clipped original" do
+        result = []
+        'reinke'.each_subtoken 2, (1..-2) do |subtoken|
+          result << subtoken
+        end
+        result.should == ['eink', 'ein', 'ei']
+      end
+    end
+  end
+
+end

data/spec/lib/extensions/symbol_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe Symbol do
-  
+
   context 'performance' do
     include Picky::Helpers::Measuring
     before(:each) do
@@ -16,7 +16,7 @@ describe Symbol do
       performance_of { @token.each_intoken { |intoken| } }.should < 0.025 # TODO Slow!
     end
   end
-  
+
   describe 'each_intoken' do
     context 'normal symbol' do
       before(:each) do
@@ -249,7 +249,7 @@ describe Symbol do
       end
     end
   end
-  
+
   describe "each_subtoken" do
     context 'normal symbol' do
       before(:each) do
@@ -471,6 +471,22 @@ describe Symbol do
         end
       end
     end
+    context "with range param" do
+      it "returns a subtoken array from a clipped original" do
+        result = []
+        :reinke.each_subtoken 2, (0..3) do |subtoken|
+          result << subtoken
+        end
+        result.should == [:rein, :rei, :re]
+      end
+      it "returns a subtoken array from a clipped original" do
+        result = []
+        :reinke.each_subtoken 2, (1..-2) do |subtoken|
+          result << subtoken
+        end
+        result.should == [:eink, :ein, :ei]
+      end
+    end
   end
-  
+
 end

data/spec/specific/regression_spec.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe "Regression" do
+
+  # # This was described by Niko
+  # # and references a case where
+  # # an attribute and the id referenced
+  # # to the same String.
+  # #
+  # context 'fun cases' do
+  #   it 'stopwords destroy ids (final: id reference on attribute)' do
+  #     index = Picky::Index.new :stopwords do
+  #       key_format :to_sym
+  #       indexing splits_text_on: /[\\\/\s\"\'\&_,;:]+/i,
+  #                stopwords: /\b(and|the|or|on|of|in|der|die|das|und|oder)\b/i
+  #       category :text
+  #     end
+  #
+  #     referenced = "this and that"
+  #
+  #     require 'ostruct'
+  #
+  #     thing = OpenStruct.new id: referenced, text: referenced
+  #
+  #     index.add thing
+  #
+  #     try = Picky::Search.new index
+  #
+  #     try.search("this").ids.should == ["this  that"]
+  #   end
+  # end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: picky
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-08 00:00:00.000000000 Z
+date: 2011-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70199588357580 !ruby/object:Gem::Requirement
+  requirement: &70307432765800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,21 +21,21 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70199588357580
+  version_requirements: *70307432765800
 - !ruby/object:Gem::Dependency
   name: picky-client
-  requirement: &70199588357080 !ruby/object:Gem::Requirement
+  requirement: &70307432765260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.4.0
+        version: 3.4.1
   type: :development
   prerelease: false
-  version_requirements: *70199588357080
+  version_requirements: *70307432765260
 - !ruby/object:Gem::Dependency
   name: rack
-  requirement: &70199588356660 !ruby/object:Gem::Requirement
+  requirement: &70307432764800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588356660
+  version_requirements: *70307432764800
 - !ruby/object:Gem::Dependency
   name: rack_fast_escape
-  requirement: &70199588356200 !ruby/object:Gem::Requirement
+  requirement: &70307432764280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588356200
+  version_requirements: *70307432764280
 - !ruby/object:Gem::Dependency
   name: text
-  requirement: &70199588379320 !ruby/object:Gem::Requirement
+  requirement: &70307432763780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588379320
+  version_requirements: *70307432763780
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70199588378900 !ruby/object:Gem::Requirement
+  requirement: &70307432763280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588378900
+  version_requirements: *70307432763280
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70199588378400 !ruby/object:Gem::Requirement
+  requirement: &70307432762740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588378400
+  version_requirements: *70307432762740
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70199588377900 !ruby/object:Gem::Requirement
+  requirement: &70307432762180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588377900
+  version_requirements: *70307432762180
 - !ruby/object:Gem::Dependency
   name: unicorn
-  requirement: &70199588377520 !ruby/object:Gem::Requirement
+  requirement: &70307432761720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588377520
+  version_requirements: *70307432761720
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70199588377040 !ruby/object:Gem::Requirement
+  requirement: &70307432761240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588377040
+  version_requirements: *70307432761240
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70199588376580 !ruby/object:Gem::Requirement
+  requirement: &70307432760800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,10 +131,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588376580
+  version_requirements: *70307432760800
 - !ruby/object:Gem::Dependency
   name: mysql
-  requirement: &70199588376060 !ruby/object:Gem::Requirement
+  requirement: &70307432760320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,7 +142,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70199588376060
+  version_requirements: *70307432760320
 description: Fast Ruby semantic text search engine with comfortable single field interface.
 email: florian.hanke+picky@gmail.com
 executables:
@@ -198,6 +198,7 @@ files:
 - lib/picky/extensions/hash.rb
 - lib/picky/extensions/module.rb
 - lib/picky/extensions/object.rb
+- lib/picky/extensions/string.rb
 - lib/picky/extensions/symbol.rb
 - lib/picky/frontend_adapters/rack.rb
 - lib/picky/generators/aliases.rb
@@ -313,6 +314,7 @@ files:
 - spec/lib/extensions/hash_spec.rb
 - spec/lib/extensions/module_spec.rb
 - spec/lib/extensions/object_spec.rb
+- spec/lib/extensions/string_spec.rb
 - spec/lib/extensions/symbol_spec.rb
 - spec/lib/frontend_adapters/rack_spec.rb
 - spec/lib/generators/aliases_spec.rb
@@ -376,6 +378,7 @@ files:
 - spec/lib/tasks/try_spec.rb
 - spec/lib/tokenizer_spec.rb
 - spec/specific/realtime_spec.rb
+- spec/specific/regression_spec.rb
 - spec/specific/speed_spec.rb
 - bin/picky
 homepage: http://florianhanke.com/picky
@@ -436,6 +439,7 @@ test_files:
 - spec/lib/extensions/hash_spec.rb
 - spec/lib/extensions/module_spec.rb
 - spec/lib/extensions/object_spec.rb
+- spec/lib/extensions/string_spec.rb
 - spec/lib/extensions/symbol_spec.rb
 - spec/lib/frontend_adapters/rack_spec.rb
 - spec/lib/generators/aliases_spec.rb
@@ -499,4 +503,5 @@ test_files:
 - spec/lib/tasks/try_spec.rb
 - spec/lib/tokenizer_spec.rb
 - spec/specific/realtime_spec.rb
+- spec/specific/regression_spec.rb
 - spec/specific/speed_spec.rb