picky 0.10.5 → 0.11.0

data/lib/picky/tokenizers/index.rb

@@ -61,8 +61,11 @@ module Tokenizers
     #
     # Override in subclasses to redefine behaviour.
     #
+    # TODO Make parametrizable! reject { |token| }
+    #
     def reject tokens
-      tokens.reject! { |token| token.to_s.size < 2 }
+      tokens.reject! &:blank?
+      # tokens.reject! { |token| token.to_s.size < 2 }
     end
     
   end

data/lib/picky/type.rb

@@ -0,0 +1,46 @@
+# This class defines the indexing and index API.
+#
+# Note: A Type holds both an Index::Type and an Indexing::Type.
+#
+class Type
+  
+  # TODO Delegation.
+  #
+  
+  attr_reader :name, :indexing, :index
+  
+  def initialize name, source, options = {}
+    @name     = name
+    @indexing = Indexing::Type.new name, source, options
+    @index    = Index::Type.new    name, options
+    
+    # Centralized registry.
+    #
+    ::Indexes.register self
+  end
+  
+  # API.
+  #
+  # TODO Spec! Doc!
+  #
+  def category name, options = {}
+    name = name.to_sym
+    
+    indexing.add_category name, options
+    index.add_category    name, options
+    
+    self
+  end
+  # def location name, options = {}
+  #   grid      = options.delete :grid
+  #   precision = options.delete :precision
+  #   
+  #   options[:index_tokenizer] ||= Tokenizers::Index.new # TODO Or a specific location tokenizer.
+  #   options[:query_tokenizer] ||= Tokenizers::Query.new # TODO Or a specific location tokenizer.
+  #   options[:source_wrapper]  ||= Sources::Wrappers::Location.new(options)
+  #   
+  #   new_category = category name, options
+  #   :source => Sources::Wrappers::Location.new(source, grid:2), :tokenizer => Tokenizers::Index.new
+  # end
+  
+end

data/lib/picky/types.rb

@@ -0,0 +1,38 @@
+# Comfortable API convenience class, splits methods to indexes.
+#
+class Types
+  
+  attr_reader :types, :type_mapping
+  
+  delegate :reload,
+           :load_from_cache,
+           :to => :@indexes
+  
+  delegate :index,
+           :generate_index_only,
+           :generate_cache_only,
+           :to => :@indexings
+  
+  def initialize
+    @types = []
+    @type_mapping = {}
+    
+    @indexes   = Index::Types.new
+    @indexings = Indexing::Types.new
+  end
+  
+  def register type
+    self.types << type
+    self.type_mapping[type.name] = type
+    
+    @indexings.register type.indexing
+    @indexes.register   type.index # TODO Even necessary?
+  end
+  
+  def [] name
+    name = name.to_sym
+    
+    self.type_mapping[name]
+  end
+  
+end

data/lib/tasks/index.rake

@@ -12,6 +12,10 @@ namespace :index do
   #   end
   # end
   
+  # TODO Make option to also use non-random.
+  #      rake index:randomly (default)
+  #      rake index:ordered
+  #
   desc "Takes a snapshot, indexes, and caches."
   task :generate, [:order] => :application do |_, options|
     randomly = (options.order == 'ordered') ? false : true

data/project_prototype/Gemfile

@@ -2,7 +2,7 @@ source :gemcutter
 
 # Gems required by Picky.
 #
-gem 'picky',            '~> 0.10.0'
+gem 'picky',            '~> 0.11.0'
 gem 'rake'
 gem 'bundler'
 gem 'rack',             '~> 1.2.1'

data/project_prototype/app/application.rb

@@ -22,19 +22,19 @@ class PickySearch < Application
                    maximum_tokens: 5, # Max amount of tokens passing into a query. 5 is the default.
                    substitutes_characters_with: CharacterSubstitution::European.new # Normalizes special user input, Ä -> Ae, ñ -> n etc.
                    
-  # Define an index. Use a database etc. source? http://github.com/floere/picky/wiki/Sources-Configuration#sources
+  # Define an index. Use a database etc. source?
+  # See http://github.com/floere/picky/wiki/Sources-Configuration#sources
   #
-  books_index = index :books,
-                      Sources::CSV.new(:title, :author, :isbn, :year, :publisher, :subjects, file: 'app/library.csv'),
-                      category(:title,
-                               similarity: Similarity::Phonetic.new(3),   # Up to three similar title word indexed (default: No similarity).
-                               partial: Partial::Substring.new(from: 1)), # Indexes substrings upwards from character 1 (default: -3),
-                                                                          # You'll find "picky" even when entering just a "p".
-                      category(:author,
-                               partial: Partial::Substring.new(from: 1)),
-                      category(:isbn,
-                               partial: Partial::None.new) # Partial substring searching on an ISBN does not make
-                                                           # much sense, neither does similarity.
+  books_index = index :books, Sources::CSV.new(:title, :author, :isbn, file: 'app/library.csv')
+  books_index.category :title,
+                       similarity: Similarity::Phonetic.new(3), # Up to three similar title word indexed (default: No similarity).
+                       partial: Partial::Substring.new(from: 1) # Indexes substrings upwards from character 1 (default: -3),
+                                                                # You'll find "picky" even when entering just a "p".
+  books_index.category :author,
+                       partial: Partial::Substring.new(from: 1)
+  books_index.category :isbn,
+                       partial: Partial::None.new # Partial substring searching on an ISBN does not make
+                                                  # much sense, neither does similarity.
   
   query_options = { :weights => { [:title, :author] => +3, [:title] => +1 } } # +/- points for ordered combinations.
   

data/spec/lib/application_spec.rb

@@ -8,9 +8,8 @@ describe Application do
     it "should run ok" do
       lambda {
         class MinimalTestApplication < Application
-          books = index :books,
-                        Sources::DB.new('SELECT id, title FROM books', :file => 'app/db.yml'),
-                        category(:title)
+          books = index :books, Sources::DB.new('SELECT id, title FROM books', :file => 'app/db.yml')
+          books.category :title
                               
           
           full = Query::Full.new books
@@ -42,12 +41,10 @@ describe Application do
                            substitutes_characters_with: CharacterSubstitution::European.new,
                            maximum_tokens:     5
           
-          books_index = index :books,
-                              Sources::DB.new('SELECT id, title, author, isbn13 as isbn FROM books', :file => 'app/db.yml'),
-                              category(:title, :similarity => Similarity::DoubleLevenshtone.new(3)), # Up to three similar title word indexed.
-                              category(:author),
-                              category(:isbn,  :partial => Partial::None.new) # Partially searching on an ISBN makes not much sense.
-                              
+          books_index = index :books, Sources::DB.new('SELECT id, title, author, isbn13 as isbn FROM books', :file => 'app/db.yml')
+          books_index.category :title, similarity: Similarity::DoubleLevenshtone.new(3) # Up to three similar title word indexed.
+          books_index.category :author
+          books_index.category :isbn, partial: Partial::None.new # Partially searching on an ISBN makes not much sense.
           
           full = Query::Full.new books_index
           live = Query::Live.new books_index

data/spec/lib/configuration/indexes_spec.rb

@@ -25,89 +25,4 @@ describe Configuration::Indexes do
     end
   end
   
-  describe "take_snapshot" do
-    before(:each) do
-      @type1 = stub :type, :name => :type1
-      @type2 = stub :type, :name => :type2
-      
-      @config.stub! :types => [@type1, @type2]
-    end
-    it "tells type to make a snapshot" do
-      @type2.should_receive(:take_snapshot).once.with
-      
-      @config.take_snapshot :type2
-    end
-  end
-  describe "index" do
-    before(:each) do
-      @type1 = stub :type, :name => :type1
-      @type2 = stub :type, :name => :type2
-      
-      @config.stub! :types => [@type1, @type2]
-    end
-    it "tells type to index" do
-      @type2.should_receive(:index).once.with
-      
-      @config.index :type2
-    end
-  end
-  describe "index_solr" do
-    before(:each) do
-      @type1 = stub :type, :name => :type1
-      @type2 = stub :type, :name => :type2
-      
-      @config.stub! :types => [@type1, @type2]
-    end
-    it "tells type to index_solr" do
-      @type2.should_receive(:index_solr).once.with
-      
-      @config.index_solr :type2
-    end
-  end
-  
-  describe "only_if_included_in" do
-    before(:each) do
-      type1 = stub :type, :name => :type1
-      type2 = stub :type, :name => :type2
-      
-      @config.stub! :types => [type1, type2]
-    end
-    context "without type names" do
-      it "calls the block" do
-        checker = stub :checker
-        checker.should_receive(:bla).twice
-        
-        @config.only_if_included_in do |type|
-          checker.bla
-        end
-      end
-      it "calls the block" do
-        checker = stub :checker
-        checker.should_receive(:bla).twice
-        
-        @config.only_if_included_in [] do |type|
-          checker.bla
-        end
-      end
-    end
-    context "with type names" do
-      it "is included" do
-        checker = stub :checker
-        checker.should_receive(:bla).once
-        
-        @config.only_if_included_in [:type2, :type3, :type4] do |type|
-          checker.bla
-        end
-      end
-      it "is not included" do
-        checker = stub :checker
-        checker.should_receive(:bla).never
-        
-        @config.only_if_included_in [:type3, :type4, :type5] do |type|
-          checker.bla
-        end
-      end
-    end
-  end
-  
 end

data/spec/lib/index/bundle_spec.rb

@@ -5,11 +5,9 @@ describe Index::Bundle do
   before(:each) do
     @category    = stub :category, :name => :some_category
     @type        = stub :type, :name => :some_type
-    @partial     = stub :partial
-    @weights     = stub :weights
     @similarity  = stub :similarity
     @index_class = Index::Bundle
-    @index       = @index_class.new :some_name, @category, @type, @partial, @weights, @similarity
+    @index       = @index_class.new :some_name, @category, @type, @similarity
   end
 
   describe 'identifier' do
@@ -63,33 +61,6 @@ describe Index::Bundle do
     end
   end
 
-  describe 'generate_derived' do
-    it 'should call two methods in order' do
-      @index.should_receive(:generate_weights).once.ordered
-      @index.should_receive(:generate_similarity).once.ordered
-
-      @index.generate_derived
-    end
-  end
-
-  describe 'generate_caches_from_memory' do
-    it 'should call two methods in order' do
-      @index.should_receive(:cache_from_memory_generation_message).once.ordered
-      @index.should_receive(:generate_derived).once.ordered
-
-      @index.generate_caches_from_memory
-    end
-  end
-
-  describe 'generate_caches_from_source' do
-    it 'should call two methods in order' do
-      @index.should_receive(:load_from_index_file).once.ordered
-      @index.should_receive(:generate_caches_from_memory).once.ordered
-
-      @index.generate_caches_from_source
-    end
-  end
-
   describe 'ids' do
     before(:each) do
       @index.instance_variable_set :@index, { :existing => :some_ids }
@@ -156,66 +127,12 @@ describe Index::Bundle do
     end
   end
   
-  describe 'dump' do
-    it 'should trigger dumps' do
-      @index.should_receive(:dump_index).once.with
-      @index.should_receive(:dump_similarity).once.with
-      @index.should_receive(:dump_weights).once.with
-      
-      @index.dump
-    end
-  end
-  
-  describe 'raise_unless_cache_exists' do
-    before(:each) do
-      @files = stub :files
-      @files.stub! :index_cache_ok? => true
-      @files.stub! :similarity_cache_ok? => true
-      @files.stub! :weights_cache_ok? => true
-      @files.stub! :index_cache_small? => false
-      @files.stub! :similarity_cache_small? => false
-      @files.stub! :weights_cache_small? => false
-      
-      @index.stub! :files => @files
-    end
-    context 'weights cache missing' do
-      before(:each) do
-        @files.stub! :weights_cache_ok? => false
-      end
-      it 'should raise' do
-        lambda do
-          @index.raise_unless_cache_exists
-        end.should raise_error("weights cache for some_name: some_type some_category missing.")
-      end
-    end
-    context 'similarity cache missing' do
-      before(:each) do
-        @files.stub! :similarity_cache_ok? => false
-      end
-      it 'should raise' do
-        lambda do
-          @index.raise_unless_cache_exists
-        end.should raise_error("similarity cache for some_name: some_type some_category missing.")
-      end
-    end
-    context 'index cache missing' do
-      before(:each) do
-        @files.stub! :index_cache_ok? => false
-      end
-      it 'should raise' do
-        lambda do
-          @index.raise_unless_cache_exists
-        end.should raise_error("index cache for some_name: some_type some_category missing.")
-      end
-    end
-  end
-  
   describe 'initialization' do
     before(:each) do
       @category = stub :category, :name => :some_category
       @type     = stub :type, :name => :some_type
       
-      @index = @index_class.new :some_name, @category, @type, :partial, :weights, :similarity
+      @index = @index_class.new :some_name, @category, @type, :similarity
     end
     it 'should initialize the index correctly' do
       @index.index.should == {}
@@ -226,15 +143,6 @@ describe Index::Bundle do
     it 'should initialize the similarity index correctly' do
       @index.similarity.should == {}
     end
-    it 'should initialize the name correctly' do
-      @index.category.should == @category
-    end
-    it 'should initialize the partial strategy correctly' do
-      @index.partial_strategy.should == :partial
-    end
-    it 'should initialize the weights strategy correctly' do
-      @index.weights_strategy.should == :weights
-    end
     it 'should initialize the similarity strategy correctly' do
       @index.similarity_strategy.should == :similarity
     end

data/spec/lib/index/category_spec.rb

@@ -3,60 +3,23 @@ require 'spec_helper'
 describe Index::Category do
 
   before(:each) do
-    @field      = stub :field, :name => :some_name
     @type       = stub :type, :name => :some_type
     @partial    = stub :partial
     @weights    = stub :weights
     @similarity = stub :similarity
-    @category   = Index::Category.new @field, @type, :partial => @partial, :weights => @weights, :similarity => @similarity
-
+    @category   = Index::Category.new :some_name, @type, :partial    => @partial,
+                                                         :weights    => @weights,
+                                                         :similarity => @similarity,
+                                                         :qualifiers => [:q, :qualifier]
+    
     @exact   = stub :exact, :dump => nil
     @category.stub! :exact => @exact
-
+    
     @partial = stub :partial, :dump => nil
     @category.stub! :partial => @partial
     @category.stub! :exclaim
   end
-
-  describe 'dump_caches' do
-    it 'should dump the exact index' do
-      @exact.should_receive(:dump).once.with
-
-      @category.dump_caches
-    end
-    it 'should dump the partial index' do
-      @partial.should_receive(:dump).once.with
-
-      @category.dump_caches
-    end
-  end
-
-  describe 'generate_derived_partial' do
-    it 'should delegate to partial' do
-      @partial.should_receive(:generate_derived).once.with
-
-      @category.generate_derived_partial
-    end
-  end
-
-  describe 'generate_derived_exact' do
-    it 'should delegate to exact' do
-      @exact.should_receive(:generate_derived).once.with
-
-      @category.generate_derived_exact
-    end
-  end
-
-  describe 'generate_indexes_from_exact_index' do
-    it 'should call three method in order' do
-      @category.should_receive(:generate_derived_exact).once.with().ordered
-      @category.should_receive(:generate_partial).once.with().ordered
-      @category.should_receive(:generate_derived_partial).once.with().ordered
-
-      @category.generate_indexes_from_exact_index
-    end
-  end
-
+  
   describe 'weight' do
     before(:each) do
       @token = stub :token, :text => :some_text
@@ -146,48 +109,6 @@ describe Index::Category do
       @category.bundle_for(token).should == @partial
     end
   end
-
-  describe 'generate_caches_from_memory' do
-    it 'should delegate to partial' do
-      @partial.should_receive(:generate_caches_from_memory).once.with
-
-      @category.generate_caches_from_memory
-    end
-  end
-
-  describe 'generate_partial' do
-    it 'should return whatever the partial generation returns' do
-      @exact.stub! :index
-      @partial.stub! :generate_partial_from => :generation_returns
-
-      @category.generate_partial
-    end
-    it 'should use the exact index to generate the partial index' do
-      exact_index = stub :exact_index
-      @exact.stub! :index => exact_index
-      @partial.should_receive(:generate_partial_from).once.with(exact_index)
-
-      @category.generate_partial
-    end
-  end
-
-  describe 'generate_caches_from_source' do
-    it 'should delegate to exact' do
-      @exact.should_receive(:generate_caches_from_source).once.with
-
-      @category.generate_caches_from_source
-    end
-  end
-
-  describe 'generate_caches' do
-    it 'should call three method in order' do
-      @category.should_receive(:generate_caches_from_source).once.with().ordered
-      @category.should_receive(:generate_partial).once.with().ordered
-      @category.should_receive(:generate_caches_from_memory).once.with().ordered
-      
-      @category.generate_caches
-    end
-  end
   
   describe 'load_from_cache' do
     it 'should call two methods' do

data/spec/lib/index/type_spec.rb

@@ -4,47 +4,35 @@ describe Index::Type do
   
   context "with categories" do
     before(:each) do
-      @some_type1 = stub :some_type1, :name => :some_type1
-      @some_type2 = stub :some_type2, :name => :some_type2
+      @categories = stub :categories
       
-      @category1 = Index::Category.new :some_category_name1, @some_type1
-      @category2 = Index::Category.new :some_category_name2, @some_type2
+      @index = Index::Type.new :some_name
+      @index.add_category :some_category_name1
+      @index.add_category :some_category_name2
       
-      @index = Index::Type.new :some_name, :some_result_type, false, @category1, @category2
-    end
-    describe "generate_caches" do
-      it "delegates to each category" do
-        @category1.should_receive(:generate_caches).once.with
-        @category2.should_receive(:generate_caches).once.with
-        
-        @index.generate_caches
-      end
+      @index.stub! :categories => @categories
     end
+
     describe "load_from_cache" do
       it "delegates to each category" do
-        @category1.should_receive(:load_from_cache).once.with
-        @category2.should_receive(:load_from_cache).once.with
+        @categories.should_receive(:load_from_cache).once.with
         
         @index.load_from_cache
       end
     end
-  end
-  
-  context "no categories" do
-    before(:each) do
-      @combinator = stub :combinator
-      Query::Combinator.stub! :new => @combinator
-      
-      @index = Index::Type.new :some_name, :some_result_type, false
-    end
-    
     describe "possible_combinations" do
       it "delegates to the combinator" do
-        @combinator.should_receive(:possible_combinations_for).once.with :some_token
+        @categories.should_receive(:possible_combinations_for).once.with :some_token
         
         @index.possible_combinations :some_token
       end
     end
   end
   
+  context "no categories" do
+    it "works" do
+      Index::Type.new :some_name
+    end
+  end
+  
 end

data/spec/lib/index/wrappers/exact_first_spec.rb

@@ -12,19 +12,19 @@ describe Index::Wrappers::ExactFirst do
   
   describe "self.wrap" do
     context "type" do
-      it "wraps each category" do
-        category = stub :category, :name => :some_category, :exact => :exact, :partial => :partial
-        
-        type = Index::Type.new :type_name, :result_type, false, category
-        
-        Index::Wrappers::ExactFirst.wrap type
-        
-        type.categories.first.should be_kind_of(Index::Wrappers::ExactFirst)
-      end
+      # FIXME
+      #
+      # it "wraps each category" do
+      #   type = Index::Type.new :type_name
+      #   type.add_category :some_category
+      #   
+      #   Index::Wrappers::ExactFirst.wrap type
+      #   
+      #   type.categories.should be_kind_of(Index::Wrappers::ExactFirst)
+      # end
       it "returns the type" do
-        category = stub :category, :name => :some_category, :exact => :exact, :partial => :partial
-        
-        type = Index::Type.new :type_name, :result_type, false, category
+        type = Index::Type.new :type_name
+        type.add_category :some_category
         
         Index::Wrappers::ExactFirst.wrap(type).should == type
       end

data/spec/lib/{index → indexing}/bundle_partial_generation_speed_spec.rb

@@ -1,12 +1,12 @@
 require 'spec_helper'
 
-describe Index::Bundle do
+describe Indexing::Bundle do
 
   before(:each) do
     @category         = stub :category, :name => :some_category
     @type             = stub :type, :name => :some_type
     @partial_strategy = Cacher::Partial::Substring.new :from => 1
-    @exact            = Index::Bundle.new :some_name, @category, @type, @partial_strategy, nil, nil
+    @exact            = Indexing::Bundle.new :some_name, @category, @type, nil, @partial_strategy, nil
   end
 
   def generate_random_keys amount