picky 0.2.2 → 0.2.3

data/lib/picky/cacher/similarity/double_levenshtone.rb

@@ -29,7 +29,7 @@ module Cacher
         codes.first.to_sym unless codes.empty?
       end
 
-      # Generates an index for the given index (in full index style).
+      # Generates an index for the given index (in exact index style).
       #
       # In the following form:
       # [:meier, :mueller, :peter, :pater] => { :MR => [:meier], :MLR => [:mueller], :PTR => [:peter, :pater] }

data/lib/picky/configuration/type.rb

@@ -21,7 +21,7 @@ module Configuration
       @after_indexing           = options[:after_indexing]
       @result_type              = options[:result_type] || name
       @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false       # TODO Move to query?
-      @solr                     = options[:solr] || nil
+      # @solr                     = options[:solr] || nil
     end
     def generate
       categories = fields.map { |field| field.generate }
@@ -35,16 +35,16 @@ module Configuration
         field.index
       end
     end
-    def solr_fields
-      solr ? fields.select { |field| !field.virtual? } : []
-    end
-    # TODO Delegate to Solr handler.
-    #
-    def index_solr
-      return unless solr
-      @indexer = Indexers::Solr.new self
-      @indexer.index
-    end
+    # def solr_fields
+    #   solr ? fields.select { |field| !field.virtual? } : []
+    # end
+    # # TODO Delegate to Solr handler.
+    # #
+    # def index_solr
+    #   return unless solr
+    #   @indexer = Indexers::Solr.new self
+    #   @indexer.index
+    # end
     # TODO Spec!
     #
     def connect_backend

data/lib/picky/index/bundle.rb

@@ -4,7 +4,7 @@ module Index
 
   # This is the ACTUAL index.
   #
-  # Handles full index, partial index, weights index, and similarity index.
+  # Handles exact index, partial index, weights index, and similarity index.
   #
   class Bundle
     
@@ -269,8 +269,8 @@ module Index
       generator = Cacher::PartialGenerator.new self.index
       self.index = generator.generate self.partial_strategy
     end
-    def generate_partial_from full_index
-      self.index = full_index
+    def generate_partial_from exact_index
+      self.index = exact_index
       self.generate_partial
       self
     end

data/lib/picky/index/category.rb

@@ -1,13 +1,13 @@
 module Index
   
-  # An index category holds a full and a partial index for a given field.
+  # An index category holds a exact and a partial index for a given field.
   #
-  # For example an index category for names holds a full and
+  # For example an index category for names holds a exact and
   # a partial index bundle for names.
   #
   class Category
     
-    attr_reader :name, :type, :full, :partial
+    attr_reader :name, :type, :exact, :partial
     
     #
     #
@@ -19,17 +19,17 @@ module Index
       weights    = options[:weights]    || Cacher::Weights::Default
       similarity = options[:similarity] || Cacher::Similarity::Default
       
-      @full    = options[:full_bundle]    || Bundle.new(:full,    self, type, Cacher::Partial::None.new, weights, similarity)
+      @exact   = options[:exact_bundle]   || Bundle.new(:exact,   self, type, Cacher::Partial::None.new, weights, similarity)
       @partial = options[:partial_bundle] || Bundle.new(:partial, self, type, partial, weights, Cacher::Similarity::None.new)
       
-      @full    = full_lambda.call(@full, @partial)    if full_lambda = options[:full_lambda]
-      @partial = partial_lambda.call(@full, @partial) if partial_lambda = options[:partial_lambda]
+      @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
+      @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
     end
     
     # Loads the index from cache.
     #
     def load_from_cache
-      full.load
+      exact.load
       partial.load
     end
     
@@ -50,16 +50,16 @@ module Index
       dump_caches
     end
     def generate_caches_from_db
-      full.generate_caches_from_db
+      exact.generate_caches_from_db
     end
     def generate_partial
-      partial.generate_partial_from full.index
+      partial.generate_partial_from exact.index
     end
     def generate_caches_from_memory
       partial.generate_caches_from_memory
     end
     def dump_caches
-      full.dump
+      exact.dump
       partial.dump
     end
     # TODO move to Kernel?
@@ -75,13 +75,13 @@ module Index
     
     # Used for testing.
     #
-    def generate_indexes_from_full_index
-      generate_derived_full
+    def generate_indexes_from_exact_index
+      generate_derived_exact
       generate_partial
       generate_derived_partial
     end
-    def generate_derived_full
-      full.generate_derived
+    def generate_derived_exact
+      exact.generate_derived
     end
     def generate_derived_partial
       partial.generate_derived
@@ -102,7 +102,7 @@ module Index
     # Returns the right index bundle for this token.
     #
     def bundle_for token
-      token.partial? ? partial : full
+      token.partial? ? partial : exact
     end
 
     #

data/lib/picky/index/combined.rb

@@ -2,10 +2,8 @@
 #
 module Index
 
-  # This index combines a full and partial index.
-  # It serves to order the results such that exact (full) hits are found first.
-  #
-  # TODO Rename full -> exact. exact/partial?
+  # This index combines an exact and partial index.
+  # It serves to order the results such that exact  hits are found first.
   #
   # TODO Need to use the right subtokens. Bake in?
   #
@@ -16,7 +14,7 @@ module Index
     delegate :similar,
              :identifier,
              :name,
-             :to => :@full
+             :to => :@exact
     delegate :type,
              :category,
              :weight,
@@ -27,19 +25,19 @@ module Index
              :load,
              :to => :@partial
     
-    # TODO initialize type_or_category # => installs itself on all full and partial
+    # TODO initialize type_or_category # => installs itself on all exact and partial
     #
-    def initialize full, partial
-      @full    = full
+    def initialize exact, partial
+      @exact   = exact
       @partial = partial
     end
     
     def ids text
-      @full.ids(text) + @partial.ids(text)
+      @exact.ids(text) + @partial.ids(text)
     end
     
     def weight text
-      [@full.weight(text) || 0, @partial.weight(text) || 0].max
+      [@exact.weight(text) || 0, @partial.weight(text) || 0].max
     end
     
   end

data/lib/picky/indexers/base.rb

@@ -35,11 +35,13 @@ module Indexers
       @field.source || raise_no_source
     end
     def raise_no_source
-      raise NoSourceSpecifiedException.new "No source given for #{@type.name}:#{@field.name}."
+      raise NoSourceSpecifiedException.new "No source given for #{@type.name}:#{@field.name}." # TODO field.identifier
     end
     
     # Selects the original id (indexed id) and a column to process. The column data is called "token".
     #
+    # Note: Puts together the parts first in an array, then releasing the array from time to time by joining.
+    #
     def process
       comma   = ?,
       newline = ?\n
@@ -47,19 +49,19 @@ module Indexers
       indexing_message
       
       File.open(search_index_file_name, 'w:binary') do |file|
+        result = []
         source.harvest(@type, @field) do |indexed_id, text|
           tokenizer.tokenize(text).each do |token_text|
-            file.write indexed_id
-            file.write comma
-            file.write token_text
-            file.write newline
+            result << indexed_id << comma << token_text << newline
           end
+          file.write(result.join) && result.clear if result.size > 100_000
         end
+        file.write result.join
       end
     end
     
     def indexing_message
-      puts "#{Time.now}: Indexing #{@type.name}:#{@field.name}:#{@field.indexed_name}."
+      puts "#{Time.now}: Indexing #{@type.name}:#{@field.name}:#{@field.indexed_name}." # TODO field.identifier
     end
     
   end

data/lib/picky/indexes.rb

@@ -71,13 +71,13 @@ module Indexes
   #
   def self.backup_caches
     each_category do |category|
-      category.full.backup
+      category.exact.backup
       category.partial.backup
     end
   end
   def self.restore_caches
     each_category do |category|
-      category.full.restore
+      category.exact.restore
       category.partial.restore
     end
   end
@@ -122,7 +122,7 @@ module Indexes
   def self.check_caches
     each do |type|
       type.categories.each do |category|
-        category.full.raise_unless_cache_exists
+        category.exact.raise_unless_cache_exists
         category.partial.raise_unless_cache_exists
       end
     end
@@ -133,7 +133,7 @@ module Indexes
   def self.clear_caches
     each do |type|
       type.categories.each do |category|
-        category.full.delete_all
+        category.exact.delete_all
         category.partial.delete_all
       end
     end
@@ -144,8 +144,8 @@ module Indexes
   # TODO Should be on type?
   #
   def self.create_directory_structure
-    each_bundle do |full, partial|
-      full.create_directory
+    each_bundle do |exact, partial|
+      exact.create_directory
       partial.create_directory
     end
   end
@@ -167,7 +167,7 @@ module Indexes
   end
   def self.each_bundle
     each_category do |category|
-      yield category.full, category.partial
+      yield category.exact, category.partial
     end
   end
   

data/lib/picky/loader.rb

@@ -121,7 +121,7 @@ module Loader
     load_relative 'indexers/field'
     load_relative 'indexers/default'
     #
-    load_relative 'indexers/solr'
+    # load_relative 'indexers/solr'
 
     # Partial index generation strategies.
     #
@@ -194,7 +194,7 @@ module Loader
     load_relative 'query/live'
     load_relative 'query/full'
     #
-    load_relative 'query/solr' # TODO
+    # load_relative 'query/solr'
     
     # Results.
     #
@@ -229,7 +229,7 @@ module Loader
     
     # Load tools.
     #
-    load_relative 'solr/schema_generator'
+    # load_relative 'solr/schema_generator'
     load_relative 'cores'
     
     # Load generation.

data/lib/picky/query/combination.rb

@@ -58,7 +58,7 @@ module Query
       [identifier, *@token.to_result]
     end
     
-    # full/title:Flarbl:flarbl
+    # exact/title:Flarbl:flarbl
     #
     def to_s
       "#{bundle.name}/#{to_result.join(':')}"

data/lib/picky/sources/db.rb

@@ -21,6 +21,8 @@ module Sources
     #  Or
     #  * The configuration as a hash.
     #
+    # TODO Do not use ActiveRecord directly.
+    #
     def create_database_adapter
       adapter_class = Class.new ActiveRecord::Base
       adapter_class.abstract_class = true

data/prototype_project/Gemfile

@@ -2,21 +2,20 @@ source :gemcutter
 
 # Gems required by Picky.
 #
-gem 'picky'
-gem 'bundler',          '>=0.9.26'
-gem 'rack',             '1.2.1'
-gem 'rack-mount',       '0.6.9'
-gem 'text',             '0.2.0'
-gem 'yajl-ruby',        '0.7.8', :require => 'yajl'
+gem 'picky',            '~> 0.2.0'
+gem 'bundler',          '>= 0.9.26'
+gem 'rack',             '~> 1.2.1'
+gem 'rack-mount',       '~> 0.6.9'
+gem 'text',             '~> 0.2.0'
+gem 'yajl-ruby',        '~> 0.7.8', :require => 'yajl'
 
-# Should be optional.
+# Should be optional, but isn't yet.
+#
+gem 'activerecord',     '~> 2.3.8', :require => 'active_record'
+
+# Optional.
 #
-gem 'activesupport',    '2.3.8', :require => 'active_support'
-gem 'activerecord',     '2.3.8', :require => 'active_record'
-gem 'rsolr',            '0.12.1'
-gem 'sunspot',          '1.1.0'
 gem 'rack_fast_escape', '2009.06.24'
-gem 'rspec'
 
 # Required by your project.
 #

data/prototype_project/app/db.yml

@@ -1,6 +1,6 @@
 # Just an example, see application.rb on how this file is used.
 #
-# Note: ActiveRecord is used.
+# Note: ActiveRecord is used. Sorry.
 #
 adapter: mysql
 host: localhost

data/spec/lib/configuration/type_spec.rb

@@ -20,22 +20,22 @@ describe Configuration::Type do
                                     :solr => :some_solr_option
   end
   
-  describe 'solr_fields' do
-    it 'should return all non-virtual fields' do
-      @type.solr_fields.should == [@field]
-    end
-  end
+  # describe 'solr_fields' do
+  #   it 'should return all non-virtual fields' do
+  #     @type.solr_fields.should == [@field]
+  #   end
+  # end
   
-  describe 'index_solr' do
-    it 'should get a new solr indexer and start it' do
-      solr = mock :solr
-      Indexers::Solr.should_receive(:new).once.with(@type).and_return solr
-      
-      solr.should_receive(:index).once.with
-      
-      @type.index_solr
-    end
-  end
+  # describe 'index_solr' do
+  #   it 'should get a new solr indexer and start it' do
+  #     solr = mock :solr
+  #     Indexers::Solr.should_receive(:new).once.with(@type).and_return solr
+  #     
+  #     solr.should_receive(:index).once.with
+  #     
+  #     @type.index_solr
+  #   end
+  # end
   
   describe 'index' do
     it 'should index each of the fields' do

data/spec/lib/index/bundle_partial_generation_speed_spec.rb

@@ -6,7 +6,7 @@ describe Index::Bundle do
     @category         = stub :category, :name => :some_category
     @type             = stub :type, :name => :some_type
     @partial_strategy = Cacher::Partial::Subtoken.new :down_to => 1
-    @full             = Index::Bundle.new :some_name, @category, @type, @partial_strategy, nil, nil
+    @exact            = Index::Bundle.new :some_name, @category, @type, @partial_strategy, nil, nil
   end
 
   def generate_random_keys amount
@@ -28,11 +28,11 @@ describe Index::Bundle do
       before(:each) do
         random_keys  = generate_random_keys 500
         random_ids   = generate_random_ids  500
-        @full.index = Hash[random_keys.zip(random_ids)]
+        @exact.index = Hash[random_keys.zip(random_ids)]
       end
       it 'should be fast' do
         performance_of do
-          @full.generate_partial
+          @exact.generate_partial
         end.should < 0.2
       end
     end

data/spec/lib/index/category_spec.rb

@@ -10,16 +10,16 @@ describe Index::Category do
     @similarity = stub :similarity
     @category   = Index::Category.new @field, @type, :partial => @partial, :weights => @weights, :similarity => @similarity
 
-    @full    = stub :full, :dump => nil
-    @category.stub! :full => @full
+    @exact   = stub :exact, :dump => nil
+    @category.stub! :exact => @exact
 
     @partial = stub :partial, :dump => nil
     @category.stub! :partial => @partial
   end
 
   describe 'dump_caches' do
-    it 'should dump the full index' do
-      @full.should_receive(:dump).once.with
+    it 'should dump the exact index' do
+      @exact.should_receive(:dump).once.with
 
       @category.dump_caches
     end
@@ -38,21 +38,21 @@ describe Index::Category do
     end
   end
 
-  describe 'generate_derived_full' do
-    it 'should delegate to full' do
-      @full.should_receive(:generate_derived).once.with
+  describe 'generate_derived_exact' do
+    it 'should delegate to exact' do
+      @exact.should_receive(:generate_derived).once.with
 
-      @category.generate_derived_full
+      @category.generate_derived_exact
     end
   end
 
-  describe 'generate_indexes_from_full_index' do
+  describe 'generate_indexes_from_exact_index' do
     it 'should call three method in order' do
-      @category.should_receive(:generate_derived_full).once.with().ordered
+      @category.should_receive(:generate_derived_exact).once.with().ordered
       @category.should_receive(:generate_partial).once.with().ordered
       @category.should_receive(:generate_derived_partial).once.with().ordered
 
-      @category.generate_indexes_from_full_index
+      @category.generate_indexes_from_exact_index
     end
   end
 
@@ -70,12 +70,12 @@ describe Index::Category do
         @category.weight @token
       end
     end
-    context 'full bundle' do
+    context 'exact bundle' do
       before(:each) do
-        @category.stub! :bundle_for => @full
+        @category.stub! :bundle_for => @exact
       end
       it 'should receive weight with the token text' do
-        @full.should_receive(:weight).once.with :some_text
+        @exact.should_receive(:weight).once.with :some_text
 
         @category.weight @token
       end
@@ -96,12 +96,12 @@ describe Index::Category do
         @category.ids @token
       end
     end
-    context 'full bundle' do
+    context 'exact bundle' do
       before(:each) do
-        @category.stub! :bundle_for => @full
+        @category.stub! :bundle_for => @exact
       end
       it 'should receive ids with the token text' do
-        @full.should_receive(:ids).once.with :some_text
+        @exact.should_receive(:ids).once.with :some_text
 
         @category.ids @token
       end
@@ -137,7 +137,7 @@ describe Index::Category do
     it 'should return the right bundle' do
       token = stub :token, :partial? => false
 
-      @category.bundle_for(token).should == @full
+      @category.bundle_for(token).should == @exact
     end
     it 'should return the right bundle' do
       token = stub :token, :partial? => true
@@ -156,23 +156,23 @@ describe Index::Category do
 
   describe 'generate_partial' do
     it 'should return whatever the partial generation returns' do
-      @full.stub! :index
+      @exact.stub! :index
       @partial.stub! :generate_partial_from => :generation_returns
 
       @category.generate_partial
     end
-    it 'should use the full index to generate the partial index' do
-      full_index = stub :full_index
-      @full.stub! :index => full_index
-      @partial.should_receive(:generate_partial_from).once.with(full_index)
+    it 'should use the exact index to generate the partial index' do
+      exact_index = stub :exact_index
+      @exact.stub! :index => exact_index
+      @partial.should_receive(:generate_partial_from).once.with(exact_index)
 
       @category.generate_partial
     end
   end
 
   describe 'generate_caches_from_db' do
-    it 'should delegate to full' do
-      @full.should_receive(:generate_caches_from_db).once.with
+    it 'should delegate to exact' do
+      @exact.should_receive(:generate_caches_from_db).once.with
 
       @category.generate_caches_from_db
     end
@@ -193,7 +193,7 @@ describe Index::Category do
   
   describe 'load_from_cache' do
     it 'should call two methods' do
-      @full.should_receive(:load).once
+      @exact.should_receive(:load).once
       @partial.should_receive(:load).once
       
       @category.load_from_cache

data/spec/lib/query/solr_spec.rb

@@ -1,51 +1,51 @@
-require 'spec_helper'
-
-describe Query::Solr do
-
-  describe 'real server' do
-    before(:each) do
-      @server = Query::Solr.new
-    end
-
-    describe 'execute' do
-      context 'error cases' do
-        before(:each) do
-          @tokens = stub :tokens
-        end
-        context 'tokens are malformed' do
-
-        end
-        context 'server returns strange values' do
-
-        end
-        context 'server raises' do
-          before(:each) do
-            @server.stub! :select => lambda { raise Solr::RequestError }
-          end
-          it 'should not fail' do
-            @tokens.stub! :to_solr_query => ''
-
-            lambda { @server.execute(@tokens) }.should_not raise_error
-          end
-        end
-      end
-    end
-  end
-
-  context 'with connected Server' do
-    before(:each) do
-      @server = stub :server
-      RSolr.stub! :connect => @server
-    end
-  end
-
-  context 'without connected server' do
-    before(:each) do
-      RSolr.should_receive(:connect).and_raise RuntimeError
-    end
-    it 'should have a nil server' do
-      Query::Solr.new(:some_index_type).server.should == nil
-    end
-  end
-
-end
+# require 'spec_helper'
+# 
+# describe Query::Solr do
+# 
+#   describe 'real server' do
+#     before(:each) do
+#       @server = Query::Solr.new
+#     end
+# 
+#     describe 'execute' do
+#       context 'error cases' do
+#         before(:each) do
+#           @tokens = stub :tokens
+#         end
+#         context 'tokens are malformed' do
+# 
+#         end
+#         context 'server returns strange values' do
+# 
+#         end
+#         context 'server raises' do
+#           before(:each) do
+#             @server.stub! :select => lambda { raise Solr::RequestError }
+#           end
+#           it 'should not fail' do
+#             @tokens.stub! :to_solr_query => ''
+# 
+#             lambda { @server.execute(@tokens) }.should_not raise_error
+#           end
+#         end
+#       end
+#     end
+#   end
+# 
+#   context 'with connected Server' do
+#     before(:each) do
+#       @server = stub :server
+#       RSolr.stub! :connect => @server
+#     end
+#   end
+# 
+#   context 'without connected server' do
+#     before(:each) do
+#       RSolr.should_receive(:connect).and_raise RuntimeError
+#     end
+#     it 'should have a nil server' do
+#       Query::Solr.new(:some_index_type).server.should == nil
+#     end
+#   end
+# 
+# end

data/spec/lib/solr/schema_generator_spec.rb

@@ -1,42 +1,42 @@
-# encoding: utf-8
-require 'spec_helper'
-
-describe Solr::SchemaGenerator do
-
-  before(:each) do
-    @types = stub :types
-    @configuration = stub :configuration, :types => @types
-    @generator = Solr::SchemaGenerator.new @configuration
-  end
-
-  describe 'bound_field_names' do
-    before(:each) do
-      @generator.stub! :combine_field_names => :some_field_names
-    end
-    it 'should bind field_names' do
-      b = @generator.bound_field_names
-
-      eval('field_names', b).should == :some_field_names
-    end
-  end
-
-  describe 'generate' do
-    before(:each) do
-      @generator.stub! :bound_field_names
-      @generator.stub! :generate_schema_for
-    end
-    it 'should receive generate_schema_for once with the result of extract_binding' do
-      @generator.stub! :bound_field_names => :some_binding
-
-      @generator.should_receive(:generate_schema_for).once.with :some_binding
-
-      @generator.generate
-    end
-    it 'should extract the binding' do
-      @generator.should_receive(:bound_field_names).once.with
-
-      @generator.generate
-    end
-  end
-
-end
+# # encoding: utf-8
+# require 'spec_helper'
+# 
+# describe Solr::SchemaGenerator do
+# 
+#   before(:each) do
+#     @types = stub :types
+#     @configuration = stub :configuration, :types => @types
+#     @generator = Solr::SchemaGenerator.new @configuration
+#   end
+# 
+#   describe 'bound_field_names' do
+#     before(:each) do
+#       @generator.stub! :combine_field_names => :some_field_names
+#     end
+#     it 'should bind field_names' do
+#       b = @generator.bound_field_names
+# 
+#       eval('field_names', b).should == :some_field_names
+#     end
+#   end
+# 
+#   describe 'generate' do
+#     before(:each) do
+#       @generator.stub! :bound_field_names
+#       @generator.stub! :generate_schema_for
+#     end
+#     it 'should receive generate_schema_for once with the result of extract_binding' do
+#       @generator.stub! :bound_field_names => :some_binding
+# 
+#       @generator.should_receive(:generate_schema_for).once.with :some_binding
+# 
+#       @generator.generate
+#     end
+#     it 'should extract the binding' do
+#       @generator.should_receive(:bound_field_names).once.with
+# 
+#       @generator.generate
+#     end
+#   end
+# 
+# end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 3
+  version: 0.2.3
 platform: ruby
 authors: 
 - Florian Hanke
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-14 00:00:00 +02:00
+date: 2010-10-15 00:00:00 +02:00
 default_executable: picky
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -84,7 +84,6 @@ files:
 - lib/picky/indexers/solr.rb
 - lib/picky/indexes.rb
 - lib/picky/initializers/ext.rb
-- lib/picky/initializers/mysql.rb
 - lib/picky/loader.rb
 - lib/picky/loggers/search.rb
 - lib/picky/query/allocation.rb

data/lib/picky/initializers/mysql.rb

@@ -1,22 +0,0 @@
-# Wrapper for the mysql adapter method execute
-# to handle the 8 hours disconnect problem.
-# (http://www.mysql.fr/search/?q=autoreconnect)
-#
-ActiveRecord::ConnectionAdapters::MysqlAdapter.module_eval do
-  def execute_with_retry_once(sql, name = nil)
-    retried = false
-    begin
-      execute_without_retry_once(sql, name)
-    rescue ActiveRecord::StatementInvalid => statement_invalid_exception
-      # Our database connection has gone away, reconnect and retry this method
-      #
-      reconnect!
-      unless retried
-        retried = true
-        retry
-      end
-    end
-  end
-
-  alias_method_chain :execute, :retry_once
-end