picky 1.5.2 → 1.5.3

data/lib/picky/internals/tokenizers/base.rb

@@ -1,18 +1,35 @@
 module Internals
-  
+
   module Tokenizers # :nodoc:all
-  
+
     # Defines tokenizing processes used both in indexing and querying.
     #
     class Base
-    
+
       # TODO Move EMPTY_STRING top level.
       #
       EMPTY_STRING = ''.freeze
-    
+
+      def to_s
+        reject_condition_location = @reject_condition.to_s[/:(\d+) \(lambda\)/, 1]
+        <<-TOKENIZER
+Removes characters:        #{@removes_characters_regexp ? "/#{@removes_characters_regexp.source}/" : '-'}
+Stopwords:                 #{@remove_stopwords_regexp ? "/#{@remove_stopwords_regexp.source}/" : '-'}
+Splits text on:            #{@splits_text_on.respond_to?(:source) ? "/#{@splits_text_on.source}/" : (@splits_text_on ? @splits_text_on : '-')}
+Removes chars after split: #{@removes_characters_after_splitting_regexp ? "/#{@removes_characters_after_splitting_regexp.source}/" : '-'}
+Normalizes words:          #{@normalizes_words_regexp_replaces ? @normalizes_words_regexp_replaces : '-'}
+Rejects tokens?            #{reject_condition_location ? "Yes, see line #{reject_condition_location} in app/application.rb" : '-'}
+Substitutes chars?         #{@substituter ? "Yes, using #{@substituter}." : '-' }
+TOKENIZER
+      end
+
       # Stopwords.
       #
+      # We only allow regexps (even if string would be okay
+      # too for gsub! - it's too hard to understand)
+      #
       def stopwords regexp
+        check_argument_in __method__, Regexp, regexp
         @remove_stopwords_regexp = regexp
       end
       def remove_stopwords text
@@ -24,36 +41,45 @@ module Internals
         return text if text.match @@non_single_stopword_regexp
         remove_stopwords text
       end
-    
+
       # Illegals.
       #
-      # TODO Should there be a legal?
+      # We only allow regexps (even if string would be okay
+      # too for gsub! - it's too hard to understand)
       #
       def removes_characters regexp
+        check_argument_in __method__, Regexp, regexp
         @removes_characters_regexp = regexp
       end
       def remove_illegals text
         text.gsub! @removes_characters_regexp, EMPTY_STRING if @removes_characters_regexp
         text
       end
-    
+
       # Splitting.
       #
-      def splits_text_on regexp
-        @splits_text_on_regexp = regexp
+      # We allow Strings and Regexps.
+      # Note: We do not test against to_str since symbols do not work with String#split.
+      #
+      def splits_text_on regexp_or_string
+        raise ArgumentError.new "#{__method__} takes a Regexp or String as argument, not a #{regexp_or_string.class}." unless Regexp === regexp_or_string || String === regexp_or_string
+        @splits_text_on = regexp_or_string
       end
       def split text
-        text.split @splits_text_on_regexp
+        text.split @splits_text_on
       end
-    
+
       # Normalizing.
       #
+      # We only allow arrays.
+      #
       def normalizes_words regexp_replaces
+        raise ArgumentError.new "#{__method__} takes an Array of replaces as argument, not a #{regexp_replaces.class}." unless regexp_replaces.respond_to?(:to_ary)
         @normalizes_words_regexp_replaces = regexp_replaces
       end
       def normalize_with_patterns text
         return text unless @normalizes_words_regexp_replaces
-      
+
         @normalizes_words_regexp_replaces.each do |regex, replace|
           # This should be sufficient
           #
@@ -62,28 +88,32 @@ module Internals
         remove_after_normalizing_illegals text
         text
       end
-    
+
       # Illegal after normalizing.
       #
+      # We only allow regexps (even if string would be okay
+      # too for gsub! - it's too hard to understand)
+      #
       def removes_characters_after_splitting regexp
+        check_argument_in __method__, Regexp, regexp
         @removes_characters_after_splitting_regexp = regexp
       end
       def remove_after_normalizing_illegals text
         text.gsub! @removes_characters_after_splitting_regexp, EMPTY_STRING if @removes_characters_after_splitting_regexp
       end
-    
+
       # Substitute Characters with this substituter.
       #
       # Default is European Character substitution.
       #
       def substitutes_characters_with substituter = CharacterSubstituters::WestEuropean.new
-        # TODO Raise if it doesn't quack substitute?
+        raise ArgumentError.new "The substitutes_characters_with option needs a character substituter, which responds to #substitute." unless substituter.respond_to?(:substitute)
         @substituter = substituter
       end
       def substitute_characters text
-        substituter?? substituter.substitute(text) : text 
+        substituter?? substituter.substitute(text) : text
       end
-    
+
       # Reject tokens after tokenizing based on the given criteria.
       #
       # Note: Currently only for indexing. TODO Redesign and write for both!
@@ -94,8 +124,14 @@ module Internals
       def reject tokens
         tokens.reject! &@reject_condition
       end
-    
-    
+
+      # Checks if the right argument type has been given.
+      #
+      def check_argument_in method, type, argument, &condition
+        raise ArgumentError.new "Application##{method} takes a #{type} as argument, not a #{argument.class}." unless type === argument
+      end
+
+
       # Returns a number of tokens, generated from the given text.
       #
       # Note:
@@ -109,10 +145,10 @@ module Internals
         tokens = tokens_for words # creating tokens / strings
                  process tokens   # processing tokens / strings
       end
-    
+
       attr_reader :substituter
       alias substituter? substituter
-    
+
       def initialize options = {}
         removes_characters options[:removes_characters]                                 if options[:removes_characters]
         contracts_expressions *options[:contracts_expressions]                          if options[:contracts_expressions]
@@ -120,16 +156,16 @@ module Internals
         normalizes_words options[:normalizes_words]                                     if options[:normalizes_words]
         removes_characters_after_splitting options[:removes_characters_after_splitting] if options[:removes_characters_after_splitting]
         substitutes_characters_with options[:substitutes_characters_with]               if options[:substitutes_characters_with]
-      
+
         # Defaults.
         #
         splits_text_on options[:splits_text_on] || /\s/
         reject_token_if &(options[:reject_token_if] || :blank?)
       end
-    
+
       # Hooks.
       #
-    
+
       # Preprocessing.
       #
       def preprocess text; end
@@ -142,7 +178,7 @@ module Internals
         reject tokens # Reject any tokens that don't meet criteria
         tokens
       end
-    
+
       # Converts words into real tokens.
       #
       def tokens_for words
@@ -158,9 +194,9 @@ module Internals
       def empty_tokens
         Internals::Query::Tokens.new
       end
-    
+
     end
-    
+
   end
-  
+
 end

data/lib/picky/internals/tokenizers/index.rb

@@ -1,20 +1,20 @@
 module Internals
 
   module Tokenizers
-  
+
     # The base indexing tokenizer.
     #
     # Override in indexing subclasses and define in configuration.
     #
     class Index < Base
-    
+
       def self.default= new_default
         @default = new_default
       end
       def self.default
         @default ||= new
       end
-    
+
       # Default indexing preprocessing hook.
       #
       # Does:
@@ -34,7 +34,7 @@ module Internals
         remove_non_single_stopwords text
         text
       end
-    
+
       # Default indexing pretokenizing hook.
       #
       # Does:
@@ -48,16 +48,16 @@ module Internals
           word
         end
       end
-    
+
       # Does not actually return a token, but a
       # symbol "token".
       #
       def token_for text
         symbolize text
       end
-    
+
     end
-    
+
   end
-  
+
 end

data/lib/picky/loader.rb

@@ -1,7 +1,7 @@
 # Loads the search engine and itself.
 #
 module Loader # :nodoc:all
-  
+
   # Reloads the whole app.
   # First itself, then the app.
   #
@@ -21,7 +21,7 @@ module Loader # :nodoc:all
     exclaim 'Loader loading itself.'
     load __FILE__
   end
-  
+
   def self.require_relative filename
     require File.join(File.dirname(__FILE__), filename)
   end
@@ -31,7 +31,7 @@ module Loader # :nodoc:all
   def self.load_internals filename_without_rb
     load File.join(File.dirname(__FILE__), "internals/#{filename_without_rb}.rb")
   end
-  
+
   def self.load_user filename
     load File.join(PICKY_ROOT, "#{filename}.rb")
   end
@@ -43,14 +43,14 @@ module Loader # :nodoc:all
       load filename
     end
   end
-  
+
   # Load the user's application.
   #
   def self.load_application
     # Add lib dir to load path.
     #
     # add_lib_dir
-    
+
     # Picky autoloading.
     #
     begin
@@ -63,25 +63,27 @@ module Loader # :nodoc:all
       load_user_lib namespaced_class_name.underscore # Try it once.
       retry
     end
-    
+
+    # Prepare the application for reload.
+    #
+    # TODO Application.prepare_for_reload
+
     # Load the user's config.
     #
     load_user 'app/logging'
     load_user 'app/application'
-    
+
     # Finalize the applications.
     #
-    # TODO Problem: Reload Routes. Throw them all away and do them again?
-    #
     Application.finalize_apps
-    
+
     # TODO Rewrite
     #
     Internals::Query::Qualifiers.instance.prepare
-    
+
     exclaim "Application #{Application.apps.map(&:name).join(', ')} loaded."
   end
-  
+
   # Loads the internal parts of the framework.
   # (Not for the user)
   #
@@ -89,7 +91,7 @@ module Loader # :nodoc:all
     # Load compiled C code.
     #
     load_internals 'ext/maybe_compile'
-    
+
     # Load extensions.
     #
     load_internals 'extensions/object'
@@ -97,24 +99,24 @@ module Loader # :nodoc:all
     load_internals 'extensions/symbol'
     load_internals 'extensions/module'
     load_internals 'extensions/hash'
-    
+
     # Requiring Helpers
     #
     load_internals 'helpers/measuring'
-    
+
     # Calculations.
     #
     load_internals 'calculations/location'
-    
+
     # Index generation strategies.
     #
     load_internals 'indexers/no_source_specified_error'
     load_internals 'indexers/serial'
-    
+
     # Generators.
     #
     load_internals 'generators/strategy'
-    
+
     # Partial index generation strategies.
     #
     load_internals 'generators/partial/strategy'
@@ -127,37 +129,37 @@ module Loader # :nodoc:all
     load_internals 'generators/weights/strategy'
     load_internals 'generators/weights/logarithmic'
     load_internals 'generators/weights/default'
-    
+
     # Similarity index generation strategies.
     #
     load_internals 'generators/similarity/strategy'
     load_internals 'generators/similarity/none'
     load_internals 'generators/similarity/double_levenshtone'
     load_internals 'generators/similarity/default'
-    
+
     # Index generators.
     #
     load_internals 'generators/base'
     load_internals 'generators/partial_generator'
     load_internals 'generators/weights_generator'
     load_internals 'generators/similarity_generator'
-    
+
     # Index store handling.
     #
     load_internals 'index/backend'
-    
+
     load_internals 'index/redis'
     load_internals 'index/redis/basic'
     load_internals 'index/redis/list_hash'
     load_internals 'index/redis/string_hash'
-    
+
     load_internals 'index/file/basic'
     load_internals 'index/file/text'
     load_internals 'index/file/marshal'
     load_internals 'index/file/json'
-    
+
     load_internals 'index/files'
-    
+
     # Indexing and Indexed things.
     #
     load_internals 'indexing/bundle/super_base' # TODO Remove.
@@ -167,68 +169,68 @@ module Loader # :nodoc:all
     load_internals 'indexing/category'
     load_internals 'indexing/categories'
     load_internals 'indexing/index'
-    
+
     load_internals 'indexed/bundle/base'
     load_internals 'indexed/bundle/memory'
     load_internals 'indexed/bundle/redis'
     load_internals 'indexed/category'
     load_internals 'indexed/categories'
     load_internals 'indexed/index'
-        
+
     # TODO Ok here?
     #
     load_internals 'indexed/wrappers/exact_first'
-    
+
     # Bundle Wrapper
     #
     load_internals 'indexed/wrappers/bundle/wrapper'
     load_internals 'indexed/wrappers/bundle/calculation'
     load_internals 'indexed/wrappers/bundle/location'
-    
+
     # Tokens.
     #
     load_internals 'query/token'
     load_internals 'query/tokens'
-    
+
     # Tokenizers types.
     #
     load_internals 'tokenizers/base'
     load_internals 'tokenizers/index'
     load_internals 'tokenizers/query'
-    
+
     # Query combinations, qualifiers, weigher.
     #
     load_internals 'query/combination'
     load_internals 'query/combinations/base'
     load_internals 'query/combinations/memory'
     load_internals 'query/combinations/redis'
-    
+
     load_internals 'query/allocation'
     load_internals 'query/allocations'
-    
+
     load_internals 'query/qualifiers'
-    
+
     load_internals 'query/weights'
-    
+
     load_internals 'query/indexes'
-    
+
     # Results.
     #
     load_internals 'results/base'
     load_internals 'results/full'
     load_internals 'results/live'
-    
+
     # Configuration.
     #
     load_internals 'configuration/index'
-    
+
     # Adapters.
     #
     load_internals 'adapters/rack/base'
     load_internals 'adapters/rack/query'
     load_internals 'adapters/rack/live_parameters'
     load_internals 'adapters/rack'
-    
+
     # Routing.
     #
     load_internals 'frontend_adapters/rack'
@@ -239,35 +241,39 @@ module Loader # :nodoc:all
     # Load harakiri.
     #
     load_relative 'rack/harakiri'
-    
+
+    # Load analyzer.
+    #
+    load_relative 'analyzer'
+
     # Character Substituters
     #
     load_relative 'character_substituters/west_european'
-    
+
     # Signal handling
     #
     load_relative 'signals'
-    
+
     # Logging.
     #
     load_relative 'loggers/search'
-    
+
     # Convenience accessors for generators.
     #
     load_relative 'generators/aliases'
-    
+
     # API.
     #
     load_relative 'index/base'
     load_relative 'index/memory'
     load_relative 'index/redis'
-    
+
     load_relative 'indexing/indexes'
     load_relative 'indexed/indexes'
-    
+
     load_relative 'index_bundle'
     load_relative 'aliases'
-    
+
     # Query.
     #
     load_relative 'query/base'
@@ -275,7 +281,7 @@ module Loader # :nodoc:all
     load_relative 'query/full'
     #
     # load_relative 'query/solr'
-    
+
     # Sources.
     #
     load_relative 'sources/base'
@@ -283,23 +289,23 @@ module Loader # :nodoc:all
     load_relative 'sources/csv'
     load_relative 'sources/delicious'
     load_relative 'sources/couch'
-    
+
     load_relative 'sources/wrappers/base'
     load_relative 'sources/wrappers/location'
-    
+
     # Interfaces
     #
     load_relative 'interfaces/live_parameters'
-    
+
     # Application.
     #
     load_relative 'application'
-    
+
     # Load tools. Load in specific case?
     #
     load_relative 'cores'
   end
-  
+
   # Loads the framework.
   #
   def self.load_framework