picky 1.4.1 → 1.4.2

data/lib/picky/internals/tokenizers/query.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+#
+module Internals
+
+  module Tokenizers
+  
+    # There are a few class methods that you can use to configure how a query works.
+    #
+    # removes_characters regexp
+    # illegal_after_normalizing regexp
+    # stopwords regexp
+    # contracts_expressions regexp, to_string
+    # splits_text_on regexp
+    # normalizes_words [[/regexp1/, 'replacement1'], [/regexp2/, 'replacement2']]
+    #
+    class Query < Base
+    
+      def self.default= new_default
+        @default = new_default
+      end
+      def self.default
+        @default ||= new
+      end
+    
+      attr_reader :maximum_tokens
+    
+      def initialize options = {}
+        super options
+        @maximum_tokens = options[:maximum_tokens] || 5
+      end
+    
+      def preprocess text
+        remove_illegals text             # Remove illegal characters
+        remove_non_single_stopwords text # remove stop words
+        text
+      end
+    
+      # Split the text and put some back together.
+      #
+      # TODO Make the same as in indexing?
+      #
+      def pretokenize text
+        split text
+      end
+    
+      # Let each token process itself.
+      # Reject, limit, and partialize tokens.
+      #
+      def process tokens
+        tokens.tokenize_with self
+        tokens.reject              # Reject any tokens that don't meet criteria
+        tokens.cap maximum_tokens  # Cut off superfluous tokens
+        tokens.partialize_last     # Set certain tokens as partial
+        tokens
+      end
+    
+      # Called by the token.
+      #
+      # TODO Perhaps move to Normalizer?
+      #
+      def normalize text
+        text = substitute_characters text # Substitute special characters
+        text.downcase!                    # Downcase all text
+        normalize_with_patterns text      # normalize
+        text.to_sym                       # symbolize
+      end
+    
+      # Returns a token for a word.
+      # The basic query tokenizer uses new tokens.
+      #
+      def token_for word
+        Internals::Query::Token.processed word
+      end
+    
+    end
+    
+  end
+  
+end

data/lib/picky/loader.rb

@@ -28,6 +28,9 @@ module Loader # :nodoc:all
   def self.load_relative filename_without_rb
     load File.join(File.dirname(__FILE__), "#{filename_without_rb}.rb")
   end
+  def self.load_internals filename_without_rb
+    load File.join(File.dirname(__FILE__), "internals/#{filename_without_rb}.rb")
+  end
   
   def self.load_user filename
     load File.join(PICKY_ROOT, "#{filename}.rb")
@@ -74,151 +77,196 @@ module Loader # :nodoc:all
     
     # TODO Rewrite
     #
-    Query::Qualifiers.instance.prepare
+    Internals::Query::Qualifiers.instance.prepare
     
     exclaim "Application #{Application.apps.map(&:name).join(', ')} loaded."
   end
   
-  # Loads the framework.
+  # Loads the internal parts of the framework.
+  # (Not for the user)
   #
-  def self.load_framework
+  def self.load_framework_internals
     # Load compiled C code.
     #
-    require_relative 'ext/maybe_compile'
+    load_internals 'ext/maybe_compile'
     
     # Load extensions.
     #
-    load_relative 'extensions/object'
-    load_relative 'extensions/array'
-    load_relative 'extensions/symbol'
-    load_relative 'extensions/module'
-    load_relative 'extensions/hash'
-    
-    # Load harakiri.
-    #
-    load_relative 'rack/harakiri'
+    load_internals 'extensions/object'
+    load_internals 'extensions/array'
+    load_internals 'extensions/symbol'
+    load_internals 'extensions/module'
+    load_internals 'extensions/hash'
     
     # Requiring Helpers
     #
-    load_relative 'helpers/measuring'
-    
-    # Character Substituters
-    #
-    load_relative 'character_substituters/west_european'
+    load_internals 'helpers/measuring'
     
     # Calculations.
     #
-    load_relative 'calculations/location'
+    load_internals 'calculations/location'
     
-    # Signal handling
-    #
-    load_relative 'signals'
-
-    # Various.
-    #
-    load_relative 'loggers/search'
-
     # Index generation strategies.
     #
-    load_relative 'indexers/no_source_specified_error'
-    load_relative 'indexers/serial'
-    #
-    # load_relative 'indexers/solr'
+    load_internals 'indexers/no_source_specified_error'
+    load_internals 'indexers/serial'
     
-    # Cacher.
+    # Generators.
     #
-    load_relative 'cacher/strategy'
+    load_internals 'generators/strategy'
     
     # Partial index generation strategies.
     #
-    load_relative 'cacher/partial/strategy'
-    load_relative 'cacher/partial/none'
-    load_relative 'cacher/partial/substring'
-    load_relative 'cacher/partial/default'
+    load_internals 'generators/partial/strategy'
+    load_internals 'generators/partial/none'
+    load_internals 'generators/partial/substring'
+    load_internals 'generators/partial/default'
 
     # Weight index generation strategies.
     #
-    load_relative 'cacher/weights/strategy'
-    load_relative 'cacher/weights/logarithmic'
-    load_relative 'cacher/weights/default'
+    load_internals 'generators/weights/strategy'
+    load_internals 'generators/weights/logarithmic'
+    load_internals 'generators/weights/default'
     
     # Similarity index generation strategies.
     #
-    load_relative 'cacher/similarity/strategy'
-    load_relative 'cacher/similarity/none'
-    load_relative 'cacher/similarity/double_levenshtone'
-    load_relative 'cacher/similarity/default'
-    
-    # Convenience accessors for generators.
-    #
-    load_relative 'cacher/convenience'
+    load_internals 'generators/similarity/strategy'
+    load_internals 'generators/similarity/none'
+    load_internals 'generators/similarity/double_levenshtone'
+    load_internals 'generators/similarity/default'
     
     # Index generators.
     #
-    load_relative 'cacher/generator'
-    load_relative 'cacher/partial_generator'
-    load_relative 'cacher/weights_generator'
-    load_relative 'cacher/similarity_generator'
+    load_internals 'generators/base'
+    load_internals 'generators/partial_generator'
+    load_internals 'generators/weights_generator'
+    load_internals 'generators/similarity_generator'
     
-    # Index file handling.
-    #
-    load_relative 'index/file/basic'
-    load_relative 'index/file/text'
-    load_relative 'index/file/marshal'
-    load_relative 'index/file/json'
-    load_relative 'index/files'
-    
-    # Indexing and Indexed things.
+    # Index store handling.
     #
-    load_relative 'index/bundle'
+    load_internals 'index/backend'
     
-    load_relative 'indexing/bundle'
-    load_relative 'indexing/category'
-    load_relative 'indexing/categories'
-    load_relative 'indexing/index'
-    load_relative 'indexing/indexes'
+    load_internals 'index/redis'
+    load_internals 'index/redis/basic'
+    load_internals 'index/redis/list_hash'
+    load_internals 'index/redis/string_hash'
     
-    load_relative 'indexed/bundle'
-    load_relative 'indexed/category'
-    load_relative 'indexed/categories'
-    load_relative 'indexed/index'
-    load_relative 'indexed/indexes'
+    load_internals 'index/file/basic'
+    load_internals 'index/file/text'
+    load_internals 'index/file/marshal'
+    load_internals 'index/file/json'
     
-    load_relative 'indexes_api'
-    load_relative 'alias_instances'
-    load_relative 'index_api'
+    load_internals 'index/files'
     
-    load_relative 'indexed/wrappers/exact_first'
+    # Indexing and Indexed things.
+    #
+    load_internals 'indexing/bundle/super_base' # TODO Remove.
+    load_internals 'indexing/bundle/base'
+    load_internals 'indexing/bundle/memory'
+    load_internals 'indexing/bundle/redis'
+    load_internals 'indexing/category'
+    load_internals 'indexing/categories'
+    load_internals 'indexing/index'
+    
+    load_internals 'indexed/bundle/base'
+    load_internals 'indexed/bundle/memory'
+    load_internals 'indexed/bundle/redis'
+    load_internals 'indexed/category'
+    load_internals 'indexed/categories'
+    load_internals 'indexed/index'
+        
+    # TODO Ok here?
+    #
+    load_internals 'indexed/wrappers/exact_first'
     
     # Bundle Wrapper
     #
-    load_relative 'indexed/wrappers/bundle/wrapper'
-    load_relative 'indexed/wrappers/bundle/calculation'
-    load_relative 'indexed/wrappers/bundle/location'
+    load_internals 'indexed/wrappers/bundle/wrapper'
+    load_internals 'indexed/wrappers/bundle/calculation'
+    load_internals 'indexed/wrappers/bundle/location'
     
     # Tokens.
     #
-    load_relative 'query/token'
-    load_relative 'query/tokens'
+    load_internals 'query/token'
+    load_internals 'query/tokens'
     
     # Tokenizers types.
     #
-    load_relative 'tokenizers/base'
-    load_relative 'tokenizers/index'
-    load_relative 'tokenizers/query'
+    load_internals 'tokenizers/base'
+    load_internals 'tokenizers/index'
+    load_internals 'tokenizers/query'
     
     # Query combinations, qualifiers, weigher.
     #
-    load_relative 'query/combination'
-    load_relative 'query/combinations'
+    load_internals 'query/combination'
+    load_internals 'query/combinations/base'
+    load_internals 'query/combinations/memory'
+    load_internals 'query/combinations/redis'
+    
+    load_internals 'query/allocation'
+    load_internals 'query/allocations'
+    
+    load_internals 'query/qualifiers'
+    
+    load_internals 'query/weights'
+    
+    load_internals 'query/indexes'
+    
+    # Results.
+    #
+    load_internals 'results/base'
+    load_internals 'results/full'
+    load_internals 'results/live'
+    
+    # Configuration.
+    #
+    load_internals 'configuration/index'
+    
+    # Adapters.
+    #
+    load_internals 'adapters/rack/base'
+    load_internals 'adapters/rack/query'
+    load_internals 'adapters/rack/live_parameters'
+    load_internals 'adapters/rack'
+    
+    # Routing.
+    #
+    load_internals 'frontend_adapters/rack'
+  end
+  # Loads the user interface parts.
+  #
+  def self.load_user_interface
+    # Load harakiri.
+    #
+    load_relative 'rack/harakiri'
+    
+    # Character Substituters
+    #
+    load_relative 'character_substituters/west_european'
+    
+    # Signal handling
+    #
+    load_relative 'signals'
+    
+    # Logging.
+    #
+    load_relative 'loggers/search'
+    
+    # Convenience accessors for generators.
+    #
+    load_relative 'generators/aliases'
     
-    load_relative 'query/allocation'
-    load_relative 'query/allocations'
+    # API.
+    #
+    load_relative 'index/base'
+    load_relative 'index/memory'
+    load_relative 'index/redis'
     
-    load_relative 'query/qualifiers'
-    load_relative 'query/weigher'
+    load_relative 'indexing/indexes'
+    load_relative 'indexed/indexes'
     
-    load_relative 'query/weights'
+    load_relative 'index_bundle'
+    load_relative 'aliases'
     
     # Query.
     #
@@ -228,12 +276,6 @@ module Loader # :nodoc:all
     #
     # load_relative 'query/solr'
     
-    # Results.
-    #
-    load_relative 'results/base'
-    load_relative 'results/full'
-    load_relative 'results/live'
-    
     # Sources.
     #
     load_relative 'sources/base'
@@ -245,30 +287,24 @@ module Loader # :nodoc:all
     load_relative 'sources/wrappers/base'
     load_relative 'sources/wrappers/location'
     
-    # Configuration.
-    #
-    load_relative 'configuration/index'
-    
     # Interfaces
     #
     load_relative 'interfaces/live_parameters'
     
-    # Adapters.
-    #
-    load_relative 'adapters/rack/base'
-    load_relative 'adapters/rack/query'
-    load_relative 'adapters/rack/live_parameters'
-    load_relative 'adapters/rack'
-    
-    # Application and routing.
+    # Application.
     #
-    load_relative 'frontend_adapters/rack'
     load_relative 'application'
     
-    # Load tools.
+    # Load tools. Load in specific case?
     #
-    # load_relative 'solr/schema_generator'
     load_relative 'cores'
   end
+  
+  # Loads the framework.
+  #
+  def self.load_framework
+    load_framework_internals
+    load_user_interface
+  end
 
 end

data/lib/picky/query/base.rb

@@ -14,35 +14,66 @@
 # * Query::Live (Same as the Full results without result ids. Useful for query result counters.)
 #
 module Query
-  
+
   # The base query class.
   #
   # Not directly instantiated. However, its methods are used by its subclasses, Full and Live.
   #
   class Base
-    
+  
     include Helpers::Measuring
-    
+  
     attr_writer   :tokenizer, :identifiers_to_remove
     attr_accessor :reduce_to_amount, :weights
-    
+  
     # Takes:
     # * A number of indexes
     # * Options hash (optional) with:
-    #   * weigher:   A weigher. Query::Weigher by default.
     #   * tokenizer: Tokenizers::Query.default by default.
     #   * weights:   A hash of weights, or a Query::Weights object.
     #
-    def initialize *index_type_definitions
-      options      = Hash === index_type_definitions.last ? index_type_definitions.pop : {}
-      indexes      = index_type_definitions.map &:indexed
-      
-      @weigher     = options[:weigher]   || Weigher.new(indexes)
-      @tokenizer   = options[:tokenizer] || Tokenizers::Query.default
+    def initialize *index_definitions
+      options      = Hash === index_definitions.last ? index_definitions.pop : {}
+
+      @indexes     = Internals::Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
+      @tokenizer   = options[:tokenizer] || Internals::Tokenizers::Query.default
       weights      = options[:weights] || Weights.new
       @weights     = Hash === weights ? Weights.new(weights) : weights
     end
     
+    # Returns the right combinations strategy for
+    # a number of query indexes.
+    #
+    # Currently it isn't possible using Memory and Redis etc.
+    # indexes in the same query index group.
+    #
+    # Picky will raise a Query::Indexes::DifferentTypesError.
+    #
+    @@mapping = {
+      Index::Memory => Internals::Query::Combinations::Memory,
+      Index::Redis  => Internals::Query::Combinations::Redis
+    }
+    def combinations_type_for index_definitions_ary
+      index_types = index_definitions_ary.map(&:class)
+      index_types.uniq!
+      raise_different(index_types) if index_types.size > 1
+      !index_types.empty? && @@mapping[*index_types] || Internals::Query::Combinations::Memory
+    end
+    # Currently it isn't possible using Memory and Redis etc.
+    # indexes in the same query index group.
+    #
+    class DifferentTypesError < StandardError
+      def initialize types
+        @types = types
+      end
+      def to_s
+        "Currently it isn't possible to mix #{@types.join(" and ")} Indexes in the same Query."
+      end
+    end
+    def raise_different index_types
+      raise DifferentTypesError.new(index_types)
+    end
+  
     # This is the main entry point for a query.
     # Use this in specs and also for running queries.
     #
@@ -55,22 +86,22 @@ module Query
     def search_with_text text, offset = 0
       search tokenized(text), offset
     end
-    
+  
     # Runs the actual search using Query::Tokens.
     #
     # Note: Internal method, use #search_with_text.
     #
     def search tokens, offset = 0
       results = nil
-      
+    
       duration = timed do
         results = execute(tokens, offset) || empty_results(offset) # TODO Does not work yet
       end
       results.duration = duration.round 6
-      
+    
       results
     end
-    
+  
     # Execute a search using Query::Tokens.
     #
     # Note: Internal method, use #search_with_text.
@@ -78,7 +109,7 @@ module Query
     def execute tokens, offset
       result_type.from offset, sorted_allocations(tokens)
     end
-    
+  
     # Returns an empty result with default values.
     #
     # Parameters:
@@ -87,7 +118,7 @@ module Query
     def empty_results offset = 0
       result_type.new offset
     end
-    
+  
     # Delegates the tokenizing to the query tokenizer.
     #
     # Parameters:
@@ -96,7 +127,7 @@ module Query
     def tokenized text
       @tokenizer.tokenize text
     end
-    
+  
     # Gets sorted allocations for the tokens.
     #
     # This generates the possible allocations, sorted.
@@ -112,28 +143,28 @@ module Query
       #
       # TODO uniq, score, sort in there
       #
-      allocations = @weigher.allocations_for tokens
-      
+      allocations = @indexes.allocations_for tokens
+    
       # Callbacks.
       #
       # TODO Reduce before sort?
       #
       reduce allocations
       remove_from allocations
-      
+    
       # Remove double allocations.
       #
       allocations.uniq
-      
+    
       # Score the allocations using weights as bias.
       #
       allocations.calculate_score weights
-      
+    
       # Sort the allocations.
       # (allocations are sorted according to score, highest to lowest)
       #
       allocations.sort
-      
+    
       # Return the allocations.
       #
       allocations
@@ -141,7 +172,7 @@ module Query
     def reduce allocations # :nodoc:
       allocations.reduce_to reduce_to_amount if reduce_to_amount
     end
-    
+  
     #
     #
     def remove_from allocations # :nodoc:
@@ -152,7 +183,7 @@ module Query
     def identifiers_to_remove # :nodoc:
       @identifiers_to_remove ||= []
     end
-    
+  
     # Display some nice information for the user.
     #
     def to_s

data/lib/picky/query/full.rb

@@ -12,7 +12,7 @@ module Query
     # Returns Results::Full as its result type.
     #
     def result_type
-      Results::Full
+      Internals::Results::Full
     end
 
   end

data/lib/picky/query/live.rb

@@ -16,7 +16,7 @@ module Query
     # Returns Results::Live as its result type.
     #
     def result_type
-      Results::Live
+      Internals::Results::Live
     end
 
   end

data/lib/picky/sources/db.rb

@@ -83,8 +83,15 @@ module Sources
       
       on_database.execute "DROP TABLE IF EXISTS #{origin}"
       on_database.execute "CREATE TABLE #{origin} AS #{select_statement}"
-      on_database.execute "ALTER TABLE #{origin} CHANGE COLUMN id indexed_id INTEGER"
-      on_database.execute "ALTER TABLE #{origin} ADD COLUMN id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT"
+      # TODO Use rename_column ASAP.
+      #
+      if on_database.adapter_name == "PostgreSQL"
+        on_database.execute "ALTER TABLE #{origin} RENAME COLUMN id TO indexed_id"
+        on_database.execute "ALTER TABLE #{origin} ADD COLUMN id SERIAL PRIMARY KEY"
+      else
+        on_database.execute "ALTER TABLE #{origin} CHANGE COLUMN id indexed_id INTEGER"
+        on_database.execute "ALTER TABLE #{origin} ADD COLUMN id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT"
+      end
       
       # Execute any special queries this type needs executed.
       #
@@ -119,10 +126,24 @@ module Sources
     #
     def get_data type, category, offset, &block # :nodoc:
       select_statement = harvest_statement_with_offset(type, category, offset)
-      database.connection.execute(select_statement).each do |indexed_id, text|
-        next unless text
-        text.force_encoding 'utf-8' # TODO Still needed? Or move to backend?
-        yield indexed_id, text
+      
+      # TODO Rewrite ASAP.
+      #
+      if database.connection.adapter_name == "PostgreSQL"
+        id_key   = 'indexed_id'
+        text_key = category.from.to_s
+        database.connection.execute(select_statement).each do |hash|
+          indexed_id, text = hash.values_at id_key, text_key
+          next unless text
+          text.force_encoding 'utf-8' # TODO Still needed? Or move to backend?
+          yield indexed_id, text
+        end
+      else
+        database.connection.execute(select_statement).each do |indexed_id, text|
+          next unless text
+          text.force_encoding 'utf-8' # TODO Still needed? Or move to backend?
+          yield indexed_id, text
+        end
       end
     end