picky 1.4.1 → 1.4.2

data/lib/picky/indexed/category.rb

@@ -1,85 +0,0 @@
-module Indexed
-  
-  # An index category holds a exact and a partial index for a given category.
-  #
-  # For example an index category for names holds a exact and
-  # a partial index bundle for names.
-  #
-  class Category
-    
-    attr_accessor :exact
-    attr_reader   :identifier, :name
-    attr_writer   :partial
-    
-    #
-    #
-    def initialize name, index, options = {}
-      @name = name
-      
-      configuration = Configuration::Index.new index, self
-      
-      @identifier    = configuration.identifier
-      
-      # TODO Push the defaults out into the index.
-      #
-      @partial_strategy = options[:partial] || Cacher::Partial::Default
-      similarity = options[:similarity] || Cacher::Similarity::Default
-      
-      @exact   = options[:exact_index_bundle]   || Bundle.new(:exact,   configuration, similarity)
-      @partial = options[:partial_index_bundle] || Bundle.new(:partial, configuration, similarity)
-      
-      @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
-      @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
-      
-      # TODO Extract?
-      #
-      Query::Qualifiers.add(configuration.category_name, generate_qualifiers_from(options) || [name])
-    end
-    
-    # TODO Move to Index.
-    #
-    def generate_qualifiers_from options
-      options[:qualifiers] || options[:qualifier] && [options[:qualifier]]
-    end
-    
-    # Loads the index from cache.
-    #
-    def load_from_cache
-      timed_exclaim "Loading index #{identifier}."
-      exact.load
-      partial.load
-    end
-    
-    # Gets the weight for this token's text.
-    #
-    def weight token
-      bundle_for(token).weight token.text
-    end
-    
-    # Gets the ids for this token's text.
-    #
-    def ids token
-      bundle_for(token).ids token.text
-    end
-    
-    # Returns the right index bundle for this token.
-    #
-    def bundle_for token
-      token.partial?? partial : exact
-    end
-    
-    # The partial strategy defines whether to really use the partial index.
-    #
-    def partial
-      @partial_strategy.use_exact_for_partial?? @exact : @partial
-    end
-    
-    #
-    #
-    def combination_for token
-      weight(token) && ::Query::Combination.new(token, self)
-    end
-    
-  end
-  
-end

data/lib/picky/indexed/index.rb

@@ -1,39 +0,0 @@
-module Indexed
-  
-  #
-  #
-  class Index
-    
-    attr_reader :name, :result_identifier, :combinator, :categories
-    
-    delegate :load_from_cache,
-             :to => :categories
-    
-    def initialize name, options = {}
-      @name                     = name
-      
-      @result_identifier        = options[:result_identifier] || name
-      ignore_unassigned_tokens  = options[:ignore_unassigned_tokens] || false # TODO Move to query, somehow.
-      
-      @categories = Categories.new ignore_unassigned_tokens: ignore_unassigned_tokens
-    end
-    
-    # TODO Doc.
-    #
-    def define_category category_name, options = {}
-      new_category = Category.new category_name, self, options
-      categories << new_category
-      new_category
-    end
-    
-    # Return the possible combinations for this token.
-    #
-    # A combination is a tuple <token, index_bundle>.
-    #
-    def possible_combinations token
-      categories.possible_combinations_for token
-    end
-    
-  end
-  
-end

data/lib/picky/indexed/wrappers/exact_first.rb

@@ -1,61 +0,0 @@
-# encoding: utf-8
-#
-module Indexed
-  
-  # TODO Spec
-  #
-  module Wrappers
-    
-    # This index combines an exact and partial index.
-    # It serves to order the results such that exact hits are found first.
-    #
-    # TODO Need to use the right subtokens. Bake in?
-    #
-    class ExactFirst < Bundle
-      
-      delegate :similar,
-               :identifier,
-               :name,
-               :to => :@exact
-      delegate :index,
-               :category,
-               :weight,
-               :generate_partial_from,
-               :generate_caches_from_memory,
-               :generate_derived,
-               :dump,
-               :load,
-               :to => :@partial
-      
-      def initialize category
-        @exact   = category.exact
-        @partial = category.partial
-      end
-      
-      def self.wrap index_or_category
-        if index_or_category.respond_to? :categories
-          wrap_each_of index_or_category.categories
-          index_or_category
-        else
-          new index_or_category
-        end
-      end
-      # TODO Do not extract categories!
-      #
-      def self.wrap_each_of categories
-        categories.categories.collect! { |category| new(category) }
-      end
-      
-      def ids text
-        @exact.ids(text) + @partial.ids(text)
-      end
-      
-      def weight text
-        [@exact.weight(text) || 0, @partial.weight(text) || 0].max
-      end
-      
-    end
-    
-  end
-  
-end

data/lib/picky/indexing/bundle.rb

@@ -1,213 +0,0 @@
-# encoding: utf-8
-#
-module Indexing # :nodoc:all
-
-  # This is the indexing bundle.
-  # It does all menial tasks that have nothing to do
-  # with the actual index running etc.
-  #
-  class Bundle < Index::Bundle
-    
-    attr_accessor :partial_strategy, :weights_strategy
-    attr_reader   :files
-    
-    # Path is in which directory the cache is located.
-    #
-    def initialize name, configuration, similarity_strategy, partial_strategy, weights_strategy
-      super name, configuration, similarity_strategy
-      
-      @partial_strategy    = partial_strategy
-      @weights_strategy    = weights_strategy
-    end
-    
-    # Generation
-    #
-    
-    # This method
-    # * loads the base index from the db
-    # * generates derived indexes
-    # * dumps all the indexes into files
-    #
-    def generate_caches_from_source
-      load_from_index_file
-      generate_caches_from_memory
-    end
-    # Generates derived indexes from the index and dumps.
-    #
-    # Note: assumes that there is something in the index
-    #
-    def generate_caches_from_memory
-      cache_from_memory_generation_message
-      generate_derived
-    end
-    def cache_from_memory_generation_message
-      timed_exclaim "CACHE FROM MEMORY #{identifier}."
-    end
-    
-    # Generates the weights and similarity from the main index.
-    #
-    def generate_derived
-      generate_weights
-      generate_similarity
-    end
-    
-    # Load the data from the db.
-    #
-    def load_from_index_file
-      load_from_index_generation_message
-      clear
-      retrieve
-    end
-    def load_from_index_generation_message
-      timed_exclaim "LOAD INDEX #{identifier}."
-    end
-    # Retrieves the prepared index data into the index.
-    #
-    # This is in preparation for generating
-    # derived indexes (like weights, similarity)
-    # and later dumping the optimized index.
-    #
-    def retrieve
-      key_format = self[:key_format] || :to_i
-      files.retrieve do |id, token|
-        initialize_index_for token
-        index[token] << id.send(key_format)
-      end
-    end
-    # Sets up a piece of the index for the given token.
-    #
-    def initialize_index_for token
-      index[token] ||= []
-    end
-    
-    # Generators.
-    #
-    # TODO Move somewhere more fitting.
-    #
-    
-    # Generates a new index (writes its index) using the
-    # partial caching strategy of this bundle.
-    #
-    def generate_partial
-      generator = Cacher::PartialGenerator.new self.index
-      self.index = generator.generate self.partial_strategy
-    end
-    # Generate a partial index from the given exact index.
-    #
-    def generate_partial_from exact_index
-      timed_exclaim "PARTIAL GENERATE #{identifier}."
-      self.index = exact_index
-      self.generate_partial
-      self
-    end
-    # Generates a new similarity index (writes its index) using the
-    # given similarity caching strategy.
-    #
-    def generate_similarity
-      generator = Cacher::SimilarityGenerator.new self.index
-      self.similarity = generator.generate self.similarity_strategy
-    end
-    # Generates a new weights index (writes its index) using the
-    # given weight caching strategy.
-    #
-    def generate_weights
-      generator = Cacher::WeightsGenerator.new self.index
-      self.weights = generator.generate self.weights_strategy
-    end
-
-    # Saves the indexes in a dump file.
-    #
-    def dump
-      dump_index
-      dump_similarity
-      dump_weights
-      dump_configuration
-    end
-    # Dumps the core index.
-    #
-    def dump_index
-      timed_exclaim "DUMP INDEX #{identifier}."
-      files.dump_index index
-    end
-    # Dumps the weights index.
-    #
-    def dump_weights
-      timed_exclaim "DUMP WEIGHTS #{identifier}."
-      files.dump_weights weights
-    end
-    # Dumps the similarity index.
-    #
-    def dump_similarity
-      timed_exclaim "DUMP SIMILARITY #{identifier}."
-      files.dump_similarity similarity
-    end
-    # Dumps the similarity index.
-    #
-    def dump_configuration
-      timed_exclaim "DUMP CONFIGURATION #{identifier}."
-      files.dump_configuration configuration
-    end
-    
-    # Alerts the user if an index is missing.
-    #
-    def raise_unless_cache_exists
-      raise_unless_index_exists
-      raise_unless_similarity_exists
-    end
-    # Alerts the user if one of the necessary indexes
-    # (core, weights) is missing.
-    #
-    def raise_unless_index_exists
-      if partial_strategy.saved?
-        warn_if_index_small
-        raise_unless_index_ok
-      end
-    end
-    # Alerts the user if the similarity
-    # index is missing (given that it's used).
-    #
-    def raise_unless_similarity_exists
-      if similarity_strategy.saved?
-        warn_if_similarity_small
-        raise_unless_similarity_ok
-      end
-    end
-    # Warns the user if the similarity index is small.
-    #
-    def warn_if_similarity_small
-      warn_cache_small :similarity if files.similarity_cache_small?
-    end
-    # Alerts the user if the similarity index is not there.
-    #
-    def raise_unless_similarity_ok
-      raise_cache_missing :similarity unless files.similarity_cache_ok?
-    end
-    
-    # TODO Spec on down.
-    #
-    
-    # Warns the user if the core or weights indexes are small.
-    #
-    def warn_if_index_small
-      warn_cache_small :index   if files.index_cache_small?
-      warn_cache_small :weights if files.weights_cache_small?
-    end
-    # Alerts the user if the core or weights indexes are not there.
-    #
-    def raise_unless_index_ok
-      raise_cache_missing :index   unless files.index_cache_ok?
-      raise_cache_missing :weights unless files.weights_cache_ok?
-    end
-    # Outputs a warning for the given cache.
-    #
-    def warn_cache_small what
-      puts "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
-    end
-    # Raises an appropriate error message for the given cache.
-    #
-    def raise_cache_missing what
-      raise "#{what} cache for #{identifier} missing."
-    end
-    
-  end
-end

data/lib/picky/indexing/categories.rb

@@ -1,38 +0,0 @@
-module Indexing
-  
-  class Categories
-    
-    attr_reader :categories
-    
-    each_delegate :index,
-                  :cache,
-                  :generate_caches,
-                  :backup_caches,
-                  :restore_caches,
-                  :check_caches,
-                  :clear_caches,
-                  :create_directory_structure,
-                  :to => :categories
-    
-    def initialize
-      @categories = []
-    end
-    
-    def << category
-      categories << category
-    end
-    
-    def find category_name
-      category_name = category_name.to_sym
-      
-      categories.each do |category|
-        next unless category.name == category_name
-        return category
-      end
-      
-      raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
-    end
-    
-  end
-  
-end

data/lib/picky/indexing/category.rb

@@ -1,117 +0,0 @@
-module Indexing
-  
-  class Category
-    
-    attr_reader :exact, :partial, :name, :configuration, :indexer
-    
-    # Mandatory params:
-    #  * name: Category name to use as identifier and file names.
-    #  * index: Index to which this category is attached to.
-    # Options:
-    #  * partial: Partial::None.new, Partial::Substring.new(from:start_char, to:up_to_char) (defaults from:-3, to:-1)
-    #  * similarity: Similarity::None.new (default), Similarity::Phonetic.new(amount_of_similarly_linked_words)
-    #  * source: Use if the category should use a different source.
-    #  * from: The source category identifier to take the data from.
-    #
-    # Advanced Options (TODO):
-    #
-    #  * weights: 
-    #  * tokenizer: 
-    #  * exact_indexing_bundle: 
-    #  * partial_indexing_bundle: 
-    #
-    def initialize name, index, options = {}
-      @name = name
-      @from = options[:from]
-      
-      # Now we have enough info to combine the index and the category.
-      #
-      @configuration = Configuration::Index.new index, self
-      
-      @tokenizer = options[:tokenizer] || Tokenizers::Index.default
-      @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer
-      
-      # TODO Push into Bundle.
-      #
-      partial    = options[:partial]    || Cacher::Partial::Default
-      weights    = options[:weights]    || Cacher::Weights::Default
-      similarity = options[:similarity] || Cacher::Similarity::Default
-      
-      @exact   = options[:exact_indexing_bundle]   || Bundle.new(:exact,   configuration, similarity, Cacher::Partial::None.new, weights)
-      @partial = options[:partial_indexing_bundle] || Bundle.new(:partial, configuration, Cacher::Similarity::None.new, partial, weights)
-    end
-    
-    delegate :identifier, :prepare_index_directory, :to => :configuration
-    delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
-    
-    def from
-      @from || name
-    end
-    
-    # TODO Spec.
-    #
-    def backup_caches
-      timed_exclaim "Backing up #{identifier}."
-      exact.backup
-      partial.backup
-    end
-    def restore_caches
-      timed_exclaim "Restoring #{identifier}."
-      exact.restore
-      partial.restore
-    end
-    def check_caches
-      timed_exclaim "Checking #{identifier}."
-      exact.raise_unless_cache_exists
-      partial.raise_unless_cache_exists
-    end
-    def clear_caches
-      timed_exclaim "Deleting #{identifier}."
-      exact.delete
-      partial.delete
-    end
-    
-    def index
-      prepare_index_directory
-      indexer.index
-    end
-    
-    # Generates all caches for this category.
-    #
-    def cache
-      prepare_index_directory
-      configure
-      generate_caches
-    end
-    # We need to set what formatting method should be used.
-    # Uses the one defined in the indexer.
-    #
-    def configure
-      key_format = indexer.key_format
-      exact[:key_format] = key_format
-      partial[:key_format] = key_format
-    end
-    def generate_caches
-      generate_caches_from_source
-      generate_partial
-      generate_caches_from_memory
-      dump_caches
-      timed_exclaim "CACHE FINISHED #{identifier}."
-    end
-    def generate_caches_from_source
-      exact.generate_caches_from_source
-    end
-    def generate_partial
-      partial.generate_partial_from exact.index
-    end
-    def generate_caches_from_memory
-      partial.generate_caches_from_memory
-    end
-    def dump_caches
-      exact.dump
-      partial.dump
-    end
-    
-  end
-  
-end

data/lib/picky/indexing/index.rb

@@ -1,55 +0,0 @@
-module Indexing
-  
-  class Index
-    
-    attr_reader :name, :source, :categories, :after_indexing
-    
-    # Delegators for indexing.
-    #
-    delegate :connect_backend,
-             :to => :source
-             
-    delegate :index,
-             :cache,
-             :generate_caches,
-             :backup_caches,
-             :restore_caches,
-             :check_caches,
-             :clear_caches,
-             :create_directory_structure,
-             :to => :categories
-    
-    def initialize name, source, options = {}
-      @name   = name
-      @source = source
-      
-      @after_indexing = options[:after_indexing]
-      
-      @categories = Categories.new
-    end
-    
-    # TODO Spec. Doc.
-    #
-    def define_category category_name, options = {}
-      options = default_category_options.merge options
-      
-      new_category = Category.new category_name, self, options
-      categories << new_category
-      new_category
-    end
-    
-    # By default, the category uses the index's source.
-    #
-    def default_category_options
-      { :source => @source }
-    end
-    
-    # Indexing.
-    #
-    def take_snapshot
-      source.take_snapshot self
-    end
-    
-  end
-  
-end

data/lib/picky/query/allocation.rb

@@ -1,82 +0,0 @@
-module Query
-  # An allocation has a number of combinations:
-  # [token, index] [other_token, other_index], ...
-  #
-  class Allocation # :nodoc:all
-
-    attr_reader :count, :ids, :score, :combinations, :result_identifier
-    
-    #
-    #
-    def initialize combinations, result_identifier
-      @combinations      = combinations
-      @result_identifier = result_identifier
-    end
-
-    def hash
-      @combinations.hash
-    end
-    def eql? other_allocation
-      true # FIXME
-      # @combinations.eql? other_allocation.combinations
-    end
-
-    # Scores its combinations and caches the result.
-    #
-    def calculate_score weights
-      @score ||= @combinations.calculate_score(weights)
-    end
-
-    # Asks the combinations for the (intersected) ids.
-    #
-    def calculate_ids
-      @combinations.ids
-    end
-
-    # This starts the searching process.
-    #
-    def process! amount, offset
-      ids    = calculate_ids
-      @count = ids.size                         # cache the count before throwing away the ids
-      @ids   = ids.slice!(offset, amount) || [] # slice out the relevant part
-    end
-
-    #
-    #
-    def keep identifiers = [] # categories
-      @combinations.keep identifiers
-    end
-    #
-    #
-    def remove identifiers = [] # categories
-      @combinations.remove identifiers
-    end
-
-    # Sort highest score first.
-    #
-    def <=> other_allocation
-       other_allocation.score <=> self.score
-    end
-
-    # Transform the allocation into result form.
-    #
-    def to_result
-      [self.result_identifier, self.score, self.count, @combinations.to_result, self.ids] if self.count > 0
-    end
-
-    # Json representation of this allocation.
-    #
-    # Note: Delegates to to_result.
-    #
-    def to_json
-      to_result.to_json
-    end
-
-    #
-    #
-    def to_s
-      "Allocation: #{to_result.join(', ')}"
-    end
-
-  end
-end