picky 4.6.3 → 4.6.4

data/lib/picky/loader.rb

@@ -28,8 +28,10 @@ module Picky
 
       # Load a file relative to this.
       #
-      def load_relative filename_without_rb
-        Kernel.load File.join(File.dirname(__FILE__), "#{filename_without_rb}.rb")
+      def load_relative *filenames_without_rb
+        filenames_without_rb.each do |filename_without_rb|
+          Kernel.load File.join(File.dirname(__FILE__), "#{filename_without_rb}.rb")
+        end
       end
 
       # Load a user file.
@@ -46,231 +48,234 @@ module Picky
         exclaim "\nBy default, Picky needs/loads the PICKY_ROOT/app.rb file as the app.\n\n"
         raise e
       end
-
-      # Loads the internal parts of the framework.
-      # (Not for the user)
+      
+      # Loads the compiled C code.
       #
-      def load_framework_internals
-        # Load compiled C code.
-        #
-        # Note: Picky already tries to compile
-        # when installing the gem.
-        #
+      # Note: Picky already tries to compile
+      # when installing the gem.
+      #
+      def load_c_code
         require_relative '../maybe_compile'
-
-        # Load extensions.
-        #
-        load_relative 'extensions/object'
-        load_relative 'extensions/array'
-        load_relative 'extensions/symbol'
-        load_relative 'extensions/string'
-        load_relative 'extensions/module'
-        load_relative 'extensions/class'
-        load_relative 'extensions/hash'
-
-        # Requiring Helpers
-        #
-        load_relative 'helpers/measuring'
-        load_relative 'helpers/indexing'
-
-        # Extension Modules
-        #
-        load_relative 'pool'
-
-        # Calculations.
-        #
-        load_relative 'calculations/location'
-
-        # Index generation strategies.
-        #
-        load_relative 'indexers/base'
-        load_relative 'indexers/serial'
-        load_relative 'indexers/parallel'
-
-        # Generators.
-        #
+      end
+      def load_extensions
+        load_relative 'extensions/object',
+                      'extensions/array',
+                      'extensions/symbol',
+                      'extensions/string',
+                      'extensions/module',
+                      'extensions/class',
+                      'extensions/hash'
+      end
+      def load_helpers
+        load_relative 'helpers/measuring',
+                      'helpers/indexing',
+                      'helpers/identification'
+      end
+      def load_index_generation_strategies
+        load_relative 'indexers/base',
+                      'indexers/serial',
+                      'indexers/parallel'
+        
         load_relative 'generators/strategy'
-
+        
         # Partial index generation strategies.
         #
-        load_relative 'generators/partial/strategy'
-        load_relative 'generators/partial/none'
-        load_relative 'generators/partial/substring'
-        load_relative 'generators/partial/postfix'
-        load_relative 'generators/partial/infix'
-        load_relative 'generators/partial/default'
-
+        load_relative 'generators/partial/strategy',
+                      'generators/partial/none',
+                      'generators/partial/substring',
+                      'generators/partial/postfix',
+                      'generators/partial/infix',
+                      'generators/partial/default'
+        
         # Weight index generation strategies.
         #
-        load_relative 'generators/weights/strategy'
-        load_relative 'generators/weights/stub'
-        load_relative 'generators/weights/dynamic'
-        load_relative 'generators/weights/constant'
-        load_relative 'generators/weights/logarithmic'
-        load_relative 'generators/weights/default'
-
+        load_relative 'generators/weights/strategy',
+                      'generators/weights/stub',
+                      'generators/weights/dynamic',
+                      'generators/weights/constant',
+                      'generators/weights/logarithmic',
+                      'generators/weights/default'
+        
         # Similarity index generation strategies.
         #
-        load_relative 'generators/similarity/strategy'
-        load_relative 'generators/similarity/none'
-        load_relative 'generators/similarity/phonetic'
-        load_relative 'generators/similarity/metaphone'
-        load_relative 'generators/similarity/double_metaphone'
-        load_relative 'generators/similarity/soundex'
-        load_relative 'generators/similarity/default'
-
-        # Index store handling.
-        #
-        load_relative 'backends/helpers/file'
-        load_relative 'backends/backend'
+        load_relative 'generators/similarity/strategy',
+                      'generators/similarity/none',
+                      'generators/similarity/phonetic',
+                      'generators/similarity/metaphone',
+                      'generators/similarity/double_metaphone',
+                      'generators/similarity/soundex',
+                      'generators/similarity/default'
+      end
+      
+      # Loads the index store handling.
+      #
+      def load_index_stores
+        load_relative 'backends/helpers/file',
+                      'backends/backend'
 
         load_relative 'backends/prepared/text'
 
-        load_relative 'backends/memory'
-        load_relative 'backends/memory/basic'
-        load_relative 'backends/memory/marshal'
-        load_relative 'backends/memory/json'
-
-        load_relative 'backends/file'
-        load_relative 'backends/file/basic'
-        load_relative 'backends/file/json'
-
-        load_relative 'backends/redis'
-        load_relative 'backends/redis/directly_manipulable'
-        load_relative 'backends/redis/basic'
-        load_relative 'backends/redis/list'
-        load_relative 'backends/redis/string'
-        load_relative 'backends/redis/float'
-
-        load_relative 'backends/sqlite'
-        load_relative 'backends/sqlite/directly_manipulable'
-        load_relative 'backends/sqlite/basic'
-        load_relative 'backends/sqlite/array'
-        load_relative 'backends/sqlite/value'
-        load_relative 'backends/sqlite/string_key_array'
-        load_relative 'backends/sqlite/integer_key_array'
-
-        # Indexing and Indexed things.
-        #
-        load_relative 'bundle'
-        load_relative 'bundle_indexing'
-        load_relative 'bundle_indexed'
-        load_relative 'bundle_realtime'
-
-        # Wrappers.
-        #
+        load_relative 'backends/memory',
+                      'backends/memory/basic',
+                      'backends/memory/marshal',
+                      'backends/memory/json'
+
+        load_relative 'backends/file',
+                      'backends/file/basic',
+                      'backends/file/json'
+
+        load_relative 'backends/redis',
+                      'backends/redis/directly_manipulable',
+                      'backends/redis/basic',
+                      'backends/redis/list',
+                      'backends/redis/string',
+                      'backends/redis/float'
+
+        load_relative 'backends/sqlite',
+                      'backends/sqlite/directly_manipulable',
+                      'backends/sqlite/basic',
+                      'backends/sqlite/array',
+                      'backends/sqlite/value',
+                      'backends/sqlite/string_key_array',
+                      'backends/sqlite/integer_key_array'
+      end
+      
+      # Indexing and Indexed things.
+      #
+      def load_indexes
+        load_relative 'bundle',
+                      'bundle_indexing',
+                      'bundle_indexed',
+                      'bundle_realtime'
+      end
+      
+      # Index wrappers.
+      #
+      def load_wrappers
         load_relative 'category/location'
 
-        load_relative 'wrappers/bundle/delegators'
-        load_relative 'wrappers/bundle/wrapper'
-        load_relative 'wrappers/bundle/calculation'
-        load_relative 'wrappers/bundle/location'
-        load_relative 'wrappers/bundle/exact_partial'
-
-        # Tokens.
-        #
-        load_relative 'query/token'
-        load_relative 'query/tokens'
-
-        # Query combinations, qualifiers, weigher.
-        #
-        load_relative 'query/combination'
-        load_relative 'query/combinations'
+        load_relative 'wrappers/bundle/delegators',
+                      'wrappers/bundle/wrapper',
+                      'wrappers/bundle/calculation',
+                      'wrappers/bundle/location',
+                      'wrappers/bundle/exact_partial'
+      end
+      
+      # Query combinations, qualifiers, weigher.
+      #
+      def load_query
+        load_relative 'query/combination',
+                      'query/combinations'
 
-        load_relative 'query/allocation'
-        load_relative 'query/allocations'
+        load_relative 'query/allocation',
+                      'query/allocations'
 
         load_relative 'query/qualifier_category_mapper'
 
         load_relative 'query/boosts'
 
-        load_relative 'query/indexes'
-        load_relative 'query/indexes_check'
+        load_relative 'query/indexes',
+                      'query/indexes_check'
+      end
+      
+      # Loads the internal parts of the framework.
+      # (Not for the user)
+      #
+      def load_framework_internals
+        load_c_code
+        load_extensions
+        load_helpers
+        load_relative 'pool'
+        load_relative 'calculations/location' # Calculations
+        load_index_generation_strategies
+        load_index_stores
+        load_indexes
+        load_wrappers
+        load_relative 'query/token' # Token related.
+        load_relative 'query/tokens'
+        load_query
+      end
+      
+      # All things API related.
+      #
+      def load_api
+        load_relative 'api/tokenizer/character_substituter',
+                      'api/search/boost'
+      end
+      
+      def load_logging
+        load_relative 'loggers/silent',
+                      'loggers/concise',
+                      'loggers/verbose',
+                      'loggers/default'
+      end
+      
+      def load_generators
+        load_relative 'generators/weights'
+        load_relative 'generators/partial'
+        load_relative 'generators/similarity'
+        load_relative 'generators/aliases'
+      end
+      
+      def load_inner_api
+        load_relative 'category',
+                      'category_indexed',
+                      'category_indexing',
+                      'category_realtime',
+                      'category_convenience'
+
+        load_relative 'categories',
+                      'categories_indexed',
+                      'categories_indexing',
+                      'categories_realtime',
+                      'categories_convenience'
+
+        load_relative 'indexes',
+                      'indexes_indexed',
+                      'indexes_indexing',
+                      'indexes_convenience'
+
+        load_relative 'index',
+                      'index_indexed',
+                      'index_indexing',
+                      'index_realtime',
+                      'index_facets',
+                      'index_convenience'
+      end
+      
+      def load_results
+        load_relative 'results',
+                      'results/exact_first'
       end
+      
+      def load_search
+        load_relative 'search',
+                      'search_facets'
+      end
+      
+      def load_interfaces
+        load_relative 'interfaces/live_parameters/master_child',
+                      'interfaces/live_parameters/unicorn'
+      end
+      
       # Loads the user interface parts.
       #
+      # TODO Move tokenizer etc.?
+      #
       def load_user_interface
-        # Load API parts.
-        #
-        load_relative 'api/tokenizer'
-        load_relative 'api/tokenizer/character_substituter'
-        load_relative 'api/source'
-        load_relative 'api/category/weight'
-        load_relative 'api/category/partial'
-        load_relative 'api/category/similarity'
-        load_relative 'api/search/boost'
-        
-        # Loggers.
-        #
-        load_relative 'loggers/silent'
-        load_relative 'loggers/concise'
-        load_relative 'loggers/verbose'
-        load_relative 'loggers/default'
-
-        # Tokenizer.
-        #
+        load_api
+        load_logging
+        load_relative 'source'
         load_relative 'tokenizer'
-
-        # Load harakiri.
-        #
         load_relative 'rack/harakiri'
-
-        # Character Substituters
-        #
         load_relative 'character_substituters/west_european'
-
-        # Convenience accessors for generators.
-        #
-        load_relative 'generators/aliases'
-
-        # API.
-        #
-        load_relative 'category'
-        load_relative 'category_indexed'
-        load_relative 'category_indexing'
-        load_relative 'category_realtime'
-        load_relative 'category_convenience'
-
-        load_relative 'categories'
-        load_relative 'categories_indexed'
-        load_relative 'categories_indexing'
-        load_relative 'categories_realtime'
-        load_relative 'categories_convenience'
-
-        load_relative 'indexes'
-        load_relative 'indexes_indexed'
-        load_relative 'indexes_indexing'
-        load_relative 'indexes_convenience'
-
-        load_relative 'index'
-        load_relative 'index_indexed'
-        load_relative 'index_indexing'
-        load_relative 'index_realtime'
-        load_relative 'index_facets'
-        load_relative 'index_convenience'
-
-        # Results.
-        #
-        load_relative 'results'
-        load_relative 'results/exact_first'
-
-        # Search.
-        #
-        load_relative 'search'
-        load_relative 'search_facets'
-
-        # Interfaces
-        #
-        load_relative 'interfaces/live_parameters/master_child'
-        load_relative 'interfaces/live_parameters/unicorn'
-
-        # Load tools. Load specifically?
-        #
+        load_generators
+        load_inner_api
+        load_results
+        load_search
+        load_interfaces
         load_relative 'scheduler'
-
-        # Load migration notices.
-        #
-        load_relative 'migrations/from_30_to_31'
+        load_relative 'migrations/from_30_to_31' # TODO Remove.
       end
 
       # Loads the framework.

data/lib/picky/query/indexes.rb

@@ -30,7 +30,18 @@ module Picky
 
         @indexes = indexes
         
-        @mapper = QualifierCategoryMapper.new indexes # TODO Move into search?
+        remap_qualifiers
+      end
+      
+      # Updates the qualifier ("qualifier:searchterm") mapping.
+      #
+      # Example:
+      #   You dynamically add a new category to an index.
+      #   To add the qualifiers to a search, you call this
+      #   method.
+      #
+      def remap_qualifiers
+        @mapper = QualifierCategoryMapper.new @indexes # TODO Move into search?
       end
 
       # Ignore the categories with these qualifiers.

data/lib/picky/search.rb

@@ -25,6 +25,7 @@ module Picky
 
     delegate :ignore,
              :only,
+             :remap_qualifiers,
              :to => :indexes
 
     # Takes:

data/lib/picky/source.rb

@@ -0,0 +1,23 @@
+module Picky
+  
+  module Source
+    extend Helpers::Identification
+    
+    # Either a thing responding to #each or a block is fine.
+    #
+    def self.from thing, nil_ok, index_name = nil
+      if thing.respond_to?(:each) || thing.respond_to?(:call)
+        thing
+      else
+        return if nil_ok
+          
+        raise ArgumentError.new(<<-ERROR)
+The source #{identifier_for(index_name)}should respond to either the method #each or
+it can be a lambda/block, returning such a source.
+ERROR
+      end
+    end
+    
+  end
+  
+end

data/lib/picky/tokenizer.rb

@@ -6,23 +6,41 @@ module Picky
   #
   class Tokenizer
 
-    extend API::Tokenizer
-
+    extend Picky::Helpers::Identification
     include API::Tokenizer::CharacterSubstituter
 
     def self.default_indexing_with options = {}
-      @indexing = extract_tokenizer options
+      @indexing = from options
     end
     def self.indexing
       @indexing ||= new
     end
 
     def self.default_searching_with options = {}
-      @searching = extract_tokenizer options
+      @searching = from options
     end
     def self.searching
       @searching ||= new
     end
+    
+    def self.from thing, index_name = nil, category_name = nil
+      return unless thing
+        
+      if thing.respond_to? :tokenize
+        thing
+      else
+        if thing.respond_to? :[]
+          Picky::Tokenizer.new thing
+        else
+          raise <<-ERROR
+indexing options #{identifier_for(index_name, category_name)}should be either
+* a Hash
+or
+* an object that responds to #tokenize(text) => [[token1, token2, ...], [original1, original2, ...]]
+ERROR
+        end
+      end
+    end
 
     def to_s
       reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
@@ -120,7 +138,7 @@ Case sensitive?     #{@case_sensitive ? "Yes." : "-"}
 
     # Reject tokens after tokenizing based on the given criteria.
     #
-    def rejects_token_if &condition
+    def rejects_token_if condition
       @reject_condition = condition
     end
     def reject tokens
@@ -161,14 +179,18 @@ Case sensitive?     #{@case_sensitive ? "Yes." : "-"}
     alias substituter? substituter
 
     def initialize options = {}
-      substitutes_characters_with options[:substitutes_characters_with] if options[:substitutes_characters_with]
-      removes_characters options[:removes_characters]                   if options[:removes_characters]
-      stopwords options[:stopwords]                                     if options[:stopwords]
-      splits_text_on options[:splits_text_on]                           || /\s/
-      normalizes_words options[:normalizes_words]                       if options[:normalizes_words]
-      max_words options[:max_words]
-      rejects_token_if &(options[:rejects_token_if]                     || :blank?)
-      case_sensitive options[:case_sensitive]                           unless options[:case_sensitive].nil?
+      options = default_options.merge options
+      options.each do |method_name, value|
+        send method_name, value unless value.nil?
+      end
+    rescue NoMethodError => e
+      raise %Q{The option "#{e.name}" is not a valid option for a Picky tokenizer.\nPlease see https://github.com/floere/picky/wiki/Indexing-configuration for valid options.}
+    end
+    def default_options
+      {
+        splits_text_on: /\s/,
+        rejects_token_if: :blank?.to_proc
+      }
     end
 
     # Returns a number of tokens, generated from the given text,

data/spec/functional/facets_spec.rb

@@ -139,7 +139,7 @@ describe 'facets' do
       it 'is fast enough' do
         performance_of {
           10.times { finder.facets(:age_category, filter: 'surname:meier name:peter') }
-        }.should < 0.00275
+        }.should < 0.003
       end
       it 'has one filtered facet' do
         # TODO Fix problems with alternative qualifiers (like :age).

data/spec/functional/remap_qualifiers_spec.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe 'qualifier remapping' do
+
+  it 'can have new qualifiers' do
+    index = Picky::Index.new :qualifier_remapping do
+      category :a
+    end
+
+    QualifierRemappingThing = Struct.new(:id, :a, :b)
+    
+    index.add QualifierRemappingThing.new(1, "a", "b")
+    
+    try = Picky::Search.new index
+    
+    # Picky finds nothing.
+    #
+    try.search('b').ids.should == []
+    
+    # Add a new category and a thing.
+    #
+    index.category :b
+    index.add QualifierRemappingThing.new(2, "c", "b")
+    
+    # It finds it.
+    #
+    try.search('b').ids.should == [2]
+    
+    # But not with qualifier!
+    #
+    try.search('b:b').ids.should == []
+    
+    # So remap the qualifiers.
+    #
+    try.remap_qualifiers
+    
+    # Now it works!
+    #
+    try.search('b:b').ids.should == [2]
+  end
+end

data/spec/functional/tokenizer_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe Picky::Tokenizer do
   describe 'examples' do
     it 'works correctly' do
-      tokenizer = described_class.new(split_words_on: /\&/, normalizes_words: [[/\&/, 'and']])
+      tokenizer = described_class.new(normalizes_words: [[/\&/, 'and']])
       
       # Is this really correct? Shouldn't we split after normalizing? 
       #

data/spec/lib/api/search/boost_spec.rb

@@ -6,7 +6,7 @@ describe Picky::API::Search::Boost do
       include Picky::API::Search::Boost
     end.new
   end
-  context 'extract_weight' do
+  context 'boost_for' do
     context 'with a Hash' do
       it 'returns a boosts object' do
         combinations = [

data/spec/lib/category_spec.rb

@@ -29,9 +29,6 @@ describe Picky::Category do
     it 'is correct' do
       category.prepared_index_path.should == 'spec/test_directory/index/test/some_index/some_category'
     end
-    it 'is correct' do
-      category.index_directory.should == 'spec/test_directory/index/test/some_index'
-    end
   end
 
   context 'tokenizer' do
@@ -62,7 +59,7 @@ describe Picky::Category do
 indexing options for some_index:some_category should be either
 * a Hash
 or
-* an object that responds to #tokenize(text) => [[token1, ...], [original1, ...]]
+* an object that responds to #tokenize(text) => [[token1, token2, ...], [original1, original2, ...]]
 ERROR
       end
     end