picky 1.3.0 → 1.3.1

data/lib/picky/index/file/text.rb

@@ -29,14 +29,15 @@ module Index
       # * id,data\n
       # * id,data\n
       #
-      # Yields an id and a symbol token.
+      # Yields an id string and a symbol token.
       #
       def retrieve
-        id, token =
+        id    = nil
+        token = nil
         ::File.open(cache_path, 'r:binary') do |file|
           file.each_line do |line|
             id, token = line.split ?,, 2
-            yield id.to_i, (token.chomp! || token).to_sym
+            yield id, (token.chomp! || token).to_sym
           end
         end
       end

data/lib/picky/indexers/serial.rb

@@ -20,6 +20,14 @@ module Indexers
       raise NoSourceSpecifiedException.new("No source given for #{@configuration.identifier}.")
     end
     
+    # Delegates the key format to the source.
+    #
+    # Default is to_i.
+    #
+    def key_format
+      @source.key_format || :to_i
+    end
+    
     # Selects the original id (indexed id) and a column to process. The column data is called "token".
     #
     # Note: Puts together the parts first in an array, then releasing the array from time to time by joining.

data/lib/picky/indexing/bundle.rb

@@ -68,9 +68,10 @@ module Indexing # :nodoc:all
     # and later dumping the optimized index.
     #
     def retrieve
+      key_format = self[:key_format] || :to_i
       files.retrieve do |id, token|
         initialize_index_for token
-        index[token] << id
+        index[token] << id.send(key_format)
       end
     end
     # Sets up a piece of the index for the given token.

data/lib/picky/indexing/category.rb

@@ -80,8 +80,17 @@ module Indexing
     #
     def cache
       prepare_index_directory
+      configure
       generate_caches
     end
+    # We need to set what formatting method should be used.
+    # Uses the one defined in the indexer.
+    #
+    def configure
+      key_format = indexer.key_format
+      exact[:key_format] = key_format
+      partial[:key_format] = key_format
+    end
     def generate_caches
       generate_caches_from_source
       generate_partial

data/lib/picky/interfaces/live_parameters.rb

@@ -22,7 +22,7 @@ module Interfaces
       #
       Thread.new do
         loop do
-          sleep 1 # TODO select
+          IO.select([@child], nil, nil, 2) or next
           result = @child.gets ';;;'
           pid, configuration_hash = eval result
           next unless Hash === configuration_hash
@@ -123,6 +123,8 @@ module Interfaces
           send :"#{key}=", new_value
         end
       rescue StandardError => e
+        # Catch any error and reraise as config error.
+        #
         raise CouldNotUpdateConfigurationError.new current_key, e.message
       end
     end

data/lib/picky/sources/base.rb

@@ -25,6 +25,8 @@ module Sources
   #
   class Base
     
+    attr_reader :key_format
+    
     # Connect to the backend.
     #
     # Called once per index/category combination before harvesting.

data/lib/picky/sources/couch.rb

@@ -9,21 +9,6 @@ module Sources
   
   # A Couch database source.
   #
-  # <b>IMPORTANT NOTE:
-  #
-  # Since Picky currently only handles integer ids (we're working on this),
-  # and CouchDB uses hexadecimal ids, this source automatically
-  # recalculates a couch id such as
-  # fa3f2577a8dbc6a91d7f9989cdffd38e
-  # into
-  # 332634873577882511228481564366832915342
-  # using String#hex.
-  #
-  # When using the integer ids in a webapp to get your
-  # objects from CouchDB, please do a Integer#to_s(16) on the
-  # ids you get from Picky before you use them to get your object from CouchDB.</b>
-  # 
-  #
   # Options:
   # * url
   # and all the options of a <tt>RestClient::Resource</tt>.
@@ -35,49 +20,6 @@ module Sources
   #
   class Couch < Base
     
-    # If your Couch DB uses UUID keys, use
-    #  Sources::Couch.new(:title, keys: Sources::Couch::UUIDKeys.new)
-    # Do not forget to reconvert the UUID Key from an integer in the client:
-    #  uuid = UUIDTools::UUID.parse_int(id)
-    #  uuid.to_s
-    #
-    class UUIDKeys
-      def initialize
-        # Tries to require the uuidtools gem.
-        #
-        begin
-          require 'uuidtools'
-        rescue LoadError
-          puts_gem_missing 'uuidtools', 'UUID keys in a CouchDB source'
-          exit 1
-        end
-      end
-      def to_i id
-        uuid = UUIDTools::UUID.parse id
-        uuid.to_i
-      end
-    end
-    
-    # If your Couch DB uses Hex keys, use
-    #  Sources::Couch.new(:title, keys: Sources::Couch::HexKeys.new)
-    # Do not forget to reconvert the Hex Key from an integer in the client:
-    #  id.to_s(16)
-    #
-    class HexKeys
-      def to_i id
-        id.hex
-      end
-    end
-    
-    # If your Couch DB uses Integer keys, use
-    #  Sources::Couch.new(:title, keys: Sources::Couch::IntegerKeys.new)
-    #
-    class IntegerKeys
-      def to_i id
-        id
-      end
-    end
-    
     #
     #
     def initialize *category_names, options
@@ -85,8 +27,12 @@ module Sources
       
       Hash === options && options[:url] || raise_no_db_given(category_names)
       @db = RestClient::Resource.new options.delete(:url), options
-      
-      @to_i_strategy = options.delete(:keys) || HexKeys.new
+    end
+    
+    # Default key format method for couch DB is to_sym. 
+    #
+    def key_format
+      :to_sym
     end
     
     # Tries to require the rest_client gem.
@@ -106,7 +52,7 @@ module Sources
     def harvest type, category
       category_name = category.from.to_s
       get_data do |doc|
-        yield @to_i_strategy.to_i(doc[@@id_key]), doc[category_name] || next
+        yield doc[@@id_key], doc[category_name] || next
       end
     end
     

data/lib/picky/sources/csv.rb

@@ -26,7 +26,7 @@ module Sources
     
     # The options that were passed into #new.
     #
-    attr_reader :csv_options
+    attr_reader :csv_options, :key_format
     
     # The data category names.
     #
@@ -36,8 +36,11 @@ module Sources
       require 'csv'
       @category_names = category_names
       
-      @csv_options    = Hash === options && options || {}
-      @file_name      = @csv_options.delete(:file) || raise_no_file_given(category_names)
+      @csv_options = Hash === options && options || {}
+      @file_name   = @csv_options.delete(:file) || raise_no_file_given(category_names)
+      
+      key_format   = options.delete :key_format
+      @key_format  = key_format && key_format.to_sym || :to_i
     end
     
     # Raises a NoCSVFileGiven exception.
@@ -51,7 +54,7 @@ module Sources
     def harvest _, category
       index = category_names.index category.from
       get_data do |ary|
-        indexed_id = ary.shift.to_i # TODO is to_i necessary?
+        indexed_id = ary.shift
         text       = ary[index]
         next unless text
         text.force_encoding 'utf-8' # TODO Still needed?

data/lib/picky/sources/delicious.rb

@@ -27,8 +27,7 @@ module Sources
     # Harvests the data to index.
     #
     def harvest _, category
-      get_data do |uid, data|
-        indexed_id = uid
+      get_data do |indexed_id, data|
         text = data[category.from]
         next unless text
         text.force_encoding 'utf-8' # TODO Still needed?

data/lib/picky/sources/wrappers/base.rb

@@ -17,7 +17,7 @@ module Sources
       
       # Default is delegation for all methods
       #
-      delegate :harvest, :connect_backend, :take_snapshot, :to => :backend
+      delegate :harvest, :connect_backend, :take_snapshot, :key_format, :to => :backend
       
     end
       

data/spec/ext/performant_spec.rb

@@ -1,7 +1,7 @@
 require File.dirname(__FILE__) + '/../spec_helper'
 
 describe Performant::Array do
-
+  
   describe "memory_efficient_intersect" do
     it "should intersect empty arrays correctly" do
       arys = [[3,4], [1,2,3], []]
@@ -51,5 +51,44 @@ describe Performant::Array do
       end.should < 0.0015
     end
   end
+  
+  describe "memory_efficient_intersect with symbols" do
+    it "should intersect empty arrays correctly" do
+      arys = [[:c,:d], [:a,:b,:c], []]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == []
+    end
+    it "should handle intermediate empty results correctly" do
+      arys = [[:e,:d], [:a,:b,:c], [:c,:d,:e,:h,:i]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == []
+    end
+    it "should intersect correctly" do
+      arys = [[:c,:d], [:a,:b,:c], [:c,:d,:e,:h,:i]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == [:c]
+    end
+    it "should intersect many arrays" do
+      arys = [[:c,:d,:e,:f,:g], [:a,:b,:c,:e,:f,:g], [:c,:d,:e,:f,:g,:h,:i], [:a,:b,:c,:d,:e,:f,:g,:h,:i,:j], [:b,:c,:e,:f,:g,:s], [:a,:b,:c,:d,:e,:f,:g,:h,:i,:j], [:b,:c,:e,:f,:g,:s]]
+
+      Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)).should == [:c,:e,:f,:g]
+    end
+    it "should be optimal for 2 small arrays of 50/10_000" do
+      arys = [(:'1'..:'50').to_a, (:'10_000'..:'20_000').to_a << 7]
+
+      # brute force
+      performance_of { Performant::Array.memory_efficient_intersect(arys.sort_by(&:size)) }.should < 0.001
+    end
+    it "should be optimal for 2 small arrays of 50/10_000" do
+      arys = [(:'1'..:'50').to_a, (:'10_000'..:'20_000').to_a << 7]
+
+      # &
+      performance_of do
+        arys.inject(arys.shift.dup) do |total, ary|
+          total & arys
+        end
+      end.should < 0.0015
+    end
+  end
 
 end

data/spec/lib/index/file/text_spec.rb

@@ -28,7 +28,7 @@ describe Index::File::Text do
     end
     it "yields split lines and returns the id and token text" do
       @file.retrieve do |id, token|
-        id.should    == 123456
+        id.should    == '123456'
         token.should == :some_nice_token
       end
     end

data/spec/lib/indexers/serial_spec.rb

@@ -13,6 +13,25 @@ describe Indexers::Serial do
     @indexer.stub! :timed_exclaim
   end
   
+  describe 'key_format' do
+    context 'source has key_format' do
+      before(:each) do
+        @source.stub! :key_format => :some_key_format
+      end
+      it 'returns what the source returns' do
+        @indexer.key_format.should == :some_key_format
+      end
+    end
+    context 'source does not have key_format' do
+      before(:each) do
+        @source.stub! :key_format => nil
+      end
+      it 'returns :to_i' do
+        @indexer.key_format.should == :to_i
+      end
+    end
+  end
+  
   describe "tokenizer" do
     it "returns the right one" do
       @indexer.tokenizer.should == @tokenizer

data/spec/lib/indexing/bundle_spec.rb

@@ -3,9 +3,9 @@ require 'spec_helper'
 describe Indexing::Bundle do
 
   before(:each) do
-    @index         = stub :index, :name => :some_index
-    @category      = stub :category, :name => :some_category
-    @configuration = Configuration::Index.new @index, @category
+    @internal_index = stub :index, :name => :some_index
+    @category       = stub :category, :name => :some_category
+    @configuration  = Configuration::Index.new @internal_index, @category
     
     @partial     = stub :partial
     @weights     = stub :weights
@@ -42,17 +42,47 @@ describe Indexing::Bundle do
     end
   end
   
-  # TODO
-  #
-  # describe 'retrieve' do
-  #   it 'should call the other methods correctly' do
-  #     results = stub :results
-  #     @index.stub! :execute_query => results
-  #     @index.should_receive(:extract).once.with results
-  #     
-  #     @index.retrieve
-  #   end
-  # end
+  describe 'retrieve' do
+    before(:each) do
+      files = stub :files
+      files.should_receive(:retrieve).once.and_yield '  1234', :some_token
+      @index.stub! :files => files
+      
+      @ary = stub :ary
+      @internal_index.should_receive(:[]).any_number_of_times.and_return @ary
+      @index.stub! :index => @internal_index
+    end
+    context 'id key format' do
+      before(:each) do
+        @index.should_receive(:[]).once.with(:key_format).and_return :to_i
+      end
+      it 'should call the other methods correctly' do
+        @ary.should_receive(:<<).once.with 1234
+        
+        @index.retrieve
+      end
+    end
+    context 'other key format' do
+      before(:each) do
+        @index.should_receive(:[]).once.with(:key_format).and_return :strip
+      end
+      it 'should call the other methods correctly' do
+        @ary.should_receive(:<<).once.with '1234'
+        
+        @index.retrieve
+      end
+    end
+    context 'no key format - default' do
+      before(:each) do
+        @index.should_receive(:[]).once.with(:key_format).and_return nil
+      end
+      it 'should call the other methods correctly' do
+        @ary.should_receive(:<<).once.with 1234
+        
+        @index.retrieve
+      end
+    end
+  end
 
   describe 'load_from_index_file' do
     it 'should call two methods in order' do

data/spec/lib/indexing/category_spec.rb

@@ -3,8 +3,9 @@ require 'spec_helper'
 describe Indexing::Category do
   
   before(:each) do
-    @index = stub :index, :name => :some_index
-    @category = Indexing::Category.new :some_category, @index, :source => :some_given_source
+    @index  = stub :index, :name => :some_index
+    @source = stub :some_given_source, :key_format => nil
+    @category = Indexing::Category.new :some_category, @index, :source => @source
   end
   context "unit specs" do
     before(:each) do

data/spec/lib/query/allocation_spec.rb

@@ -99,6 +99,32 @@ describe Query::Allocation do
         @allocation.process!(20, 10).should == []
       end
     end
+    context 'with symbol ids' do
+      before(:each) do
+        @allocation.stub! :calculate_ids => [:a,:b,:c,:d,:e,:f,:g,:h,:i,:j]
+      end
+      it 'should process right' do
+        @allocation.process!(0, 0).should == []
+      end
+      it 'should process right' do
+        @allocation.process!(0, 10).should == []
+      end
+      it 'should process right' do
+        @allocation.process!(5, 0).should == [:a,:b,:c,:d,:e]
+      end
+      it 'should process right' do
+        @allocation.process!(5, 5).should == [:f,:g,:h,:i,:j]
+      end
+      it 'should process right' do
+        @allocation.process!(20, 0).should == [:a,:b,:c,:d,:e,:f,:g,:h,:i,:j]
+      end
+      it 'should process right' do
+        @allocation.process!(20, 5).should == [:f,:g,:h,:i,:j]
+      end
+      it 'should process right' do
+        @allocation.process!(20, 10).should == []
+      end
+    end
   end
 
   describe 'to_result' do

data/spec/lib/query/allocations_spec.rb

@@ -69,20 +69,39 @@ describe Query::Allocations do
   end
 
   describe 'ids' do
-    before(:each) do
-      @allocation1 = stub :allocation1, :ids => [1, 2, 3, 4]
-      @allocation2 = stub :allocation2, :ids => [5, 6, 7]
-      @allocation3 = stub :allocation3, :ids => [8, 9]
-      @allocations = Query::Allocations.new [@allocation1, @allocation2, @allocation3]
-    end
-    it 'should return the right amount of ids' do
-      @allocations.ids(0).should == []
-    end
-    it 'should return the right amount of ids' do
-      @allocations.ids(6).should == [1,2,3,4,5,6]
+    context 'integers' do
+      before(:each) do
+        @allocation1 = stub :allocation1, :ids => [1, 2, 3, 4]
+        @allocation2 = stub :allocation2, :ids => [5, 6, 7]
+        @allocation3 = stub :allocation3, :ids => [8, 9]
+        @allocations = Query::Allocations.new [@allocation1, @allocation2, @allocation3]
+      end
+      it 'should return the right amount of ids' do
+        @allocations.ids(0).should == []
+      end
+      it 'should return the right amount of ids' do
+        @allocations.ids(6).should == [1,2,3,4,5,6]
+      end
+      it 'should return the right amount of ids' do
+        @allocations.ids.should == [1,2,3,4,5,6,7,8,9]
+      end
     end
-    it 'should return the right amount of ids' do
-      @allocations.ids.should == [1,2,3,4,5,6,7,8,9]
+    context 'symbols' do
+      before(:each) do
+        @allocation1 = stub :allocation1, :ids => [:a, :b, :c, :d]
+        @allocation2 = stub :allocation2, :ids => [:e, :f, :g]
+        @allocation3 = stub :allocation3, :ids => [:h, :i]
+        @allocations = Query::Allocations.new [@allocation1, @allocation2, @allocation3]
+      end
+      it 'should return the right amount of ids' do
+        @allocations.ids(0).should == []
+      end
+      it 'should return the right amount of ids' do
+        @allocations.ids(6).should == [:a,:b,:c,:d,:e,:f]
+      end
+      it 'should return the right amount of ids' do
+        @allocations.ids.should == [:a,:b,:c,:d,:e,:f,:g,:h,:i]
+      end
     end
   end
 

data/spec/lib/query/combinations_spec.rb

@@ -127,6 +127,13 @@ describe 'Query::Combinations' do
 
       @combinations.ids.should == (1..10).to_a
     end
+    it "should intersect symbol_keys correctly" do
+      @combination1.should_receive(:ids).once.with.and_return (:'00001'..:'10000').to_a
+      @combination2.should_receive(:ids).once.with.and_return (:'00001'..:'00100').to_a
+      @combination3.should_receive(:ids).once.with.and_return (:'00001'..:'00010').to_a
+
+      @combinations.ids.should == (:'00001'..:'0010').to_a
+    end
     it "should intersect correctly when intermediate intersect result is empty" do
       @combination1.should_receive(:ids).once.with.and_return (1..100_000).to_a
       @combination2.should_receive(:ids).once.with.and_return (11..100).to_a

data/spec/lib/sources/couch_spec.rb

@@ -2,36 +2,11 @@ require 'spec_helper'
 
 describe Sources::Couch do
   
-  describe 'UUIDKeys' do
-    before(:each) do
-      @keys = Sources::Couch::UUIDKeys.new
-    end
-    it 'converts uuids' do
-      @keys.to_i('550e8400-e29b-41d4-a716-446655440000').should == 113059749145936325402354257176981405696
-    end
-  end
-  describe 'HexKeys' do
-    before(:each) do
-      @keys = Sources::Couch::HexKeys.new
-    end
-    it 'converts uuids' do
-      @keys.to_i('7f').should == 127
-    end
-  end
-  describe 'IntegerKeys' do
-    before(:each) do
-      @keys = Sources::Couch::IntegerKeys.new
-    end
-    it 'converts uuids' do
-      @keys.to_i('123').should == '123'
-    end
-  end
-  
   describe 'special keys' do
     context 'uuid keys' do
       context "with database" do
         before(:each) do
-          @source = Sources::Couch.new :a, :b, :c, url: 'http://localhost:5984/picky', keys: Sources::Couch::UUIDKeys.new
+          @source = Sources::Couch.new :a, :b, :c, url: 'http://localhost:5984/picky'
           RestClient::Request.should_receive(:execute).any_number_of_times.and_return %{{"rows":[{"doc":{"_id":"550e8400-e29b-41d4-a716-446655440000","a":"a data","b":"b data","c":"c data"}}]}}
         end
 
@@ -39,7 +14,7 @@ describe Sources::Couch do
           it "yields the right data" do
             field = stub :b, :from => :b
             @source.harvest :anything, field do |id, token|
-              id.should    eql(113059749145936325402354257176981405696) 
+              id.should    eql('550e8400-e29b-41d4-a716-446655440000') 
               token.should eql('b data')
             end.should have(1).item
           end
@@ -57,7 +32,7 @@ describe Sources::Couch do
     context 'integer keys' do
       context "with database" do
         before(:each) do
-          @source = Sources::Couch.new :a, :b, :c, url: 'http://localhost:5984/picky', keys: Sources::Couch::IntegerKeys.new
+          @source = Sources::Couch.new :a, :b, :c, url: 'http://localhost:5984/picky'
           RestClient::Request.should_receive(:execute).any_number_of_times.and_return %{{"rows":[{"doc":{"_id":"123","a":"a data","b":"b data","c":"c data"}}]}}
         end
 
@@ -99,7 +74,7 @@ describe Sources::Couch do
         it "yields the right data" do
           field = stub :b, :from => :b
           @source.harvest :anything, field do |id, token|
-            id.should    eql(127) 
+            id.should    eql('7f') 
             token.should eql('b data')
           end.should have(1).item
         end

data/spec/lib/sources/csv_spec.rb

@@ -35,7 +35,7 @@ describe Sources::CSV do
   end
   context "with file" do
     before(:each) do
-      ::CSV.should_receive(:foreach).any_number_of_times.and_yield ['7', 'a data', 'b data', 'c data']
+      ::CSV.should_receive(:foreach).any_number_of_times.and_yield [' 7', 'a data', 'b data', 'c data']
     end
     context 'without separator' do
       before(:each) do
@@ -45,14 +45,34 @@ describe Sources::CSV do
         it "should yield the right data" do
           field = stub :b, :from => :b
           @source.harvest :anything, field do |id, token|
-            [id, token].should == [7, 'b data']
+            [id, token].should == [' 7', 'b data']
           end
         end
       end
       describe "get_data" do
         it "should yield each line" do
           @source.get_data do |data|
-            data.should == ['7', 'a data', 'b data', 'c data']
+            data.should == [' 7', 'a data', 'b data', 'c data']
+          end
+        end
+      end
+    end
+    context 'with key_format method' do
+      before(:each) do
+        @source = Sources::CSV.new :a, :b, :c, :file => :some_file, :key_format => :strip
+      end
+      describe "harvest" do
+        it "should yield the right data" do
+          field = stub :b, :from => :b
+          @source.harvest :anything, field do |id, token|
+            [id, token].should == [' 7', 'b data']
+          end
+        end
+      end
+      describe "get_data" do
+        it "should yield each line" do
+          @source.get_data do |data|
+            data.should == [' 7', 'a data', 'b data', 'c data']
           end
         end
       end

data/spec/specific/speed_spec.rb

@@ -1,7 +1,53 @@
+# encoding: utf-8
+#
 require File.dirname(__FILE__) + '/../spec_helper'
 
 describe "Speccing Ruby for speed" do
-  describe "various versions for allocation id concatenating" do
+  describe "various versions for allocation id concatenating – with symbols" do
+    before(:each) do
+      @allocs = [:hello, :speed, :test]
+      @ids = {
+        :hello => (:'000_001'..:'100_000').to_a,
+        :speed => (:'0_001'..:'5_000').to_a,
+        :test => (:'0_001'..:'1_000').to_a
+      }
+    end
+    describe "+" do
+      it "should be fast" do
+        performance_of do
+          @allocs.inject([]) do |total, alloc|
+            total + @ids[alloc]
+          end
+        end.should < 0.0025
+      end
+    end
+    describe "map and flatten!(1)" do
+      it "should be fast" do
+        performance_of do
+          @allocs.map { |alloc| @ids[alloc] }.flatten!(1)
+        end.should < 0.02
+      end
+    end
+    describe "<< and flatten!(1)" do
+      it "should be fast" do
+        performance_of do
+          @allocs.inject([]) do |total, alloc|
+            total << @ids[alloc]
+          end.flatten!(1)
+        end.should < 0.02
+      end
+    end
+    describe "<< and flatten!" do
+      it "should be fast" do
+        performance_of do
+          @allocs.inject([]) do |total, alloc|
+            total << @ids[alloc]
+          end.flatten!
+        end.should < 0.02
+      end
+    end
+  end
+  describe "various versions for allocation id concatenating – with integers" do
     before(:each) do
       @allocs = [:hello, :speed, :test]
       @ids = {

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 1
   - 3
-  - 0
-  version: 1.3.0
+  - 1
+  version: 1.3.1
 platform: ruby
 authors: 
 - Florian Hanke
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-27 00:00:00 +01:00
+date: 2011-01-30 00:00:00 +01:00
 default_executable: picky
 dependencies: 
 - !ruby/object:Gem::Dependency