picky 2.5.2 → 2.6.0

data/lib/picky/internals/indexing/bundle/super_base.rb

@@ -1,65 +0,0 @@
-module Internals
-
-  # TODO Merge into Base, extract common with Indexed::Base.
-  #
-  module Indexing # :nodoc:all
-    # A Bundle is a number of indexes
-    # per [index, category] combination.
-    #
-    # At most, there are three indexes:
-    # * *core* index (always used)
-    # * *weights* index (always used)
-    # * *similarity* index (used with similarity)
-    #
-    # In Picky, indexing is separated from the index
-    # handling itself through a parallel structure.
-    #
-    # Both use methods provided by this base class, but
-    # have very different goals:
-    #
-    # * *Indexing*::*Bundle* is just concerned with creating index files
-    #   and providing helper functions to e.g. check the indexes.
-    #
-    # * *Index*::*Bundle* is concerned with loading these index files into
-    #   memory and looking up search data as fast as possible.
-    #
-    module Bundle
-
-      class SuperBase
-
-        attr_reader   :identifier, :files
-        attr_accessor :index, :weights, :similarity, :configuration, :similarity_strategy
-
-        delegate :clear,    :to => :index
-        delegate :[], :[]=, :to => :configuration
-
-        def initialize name, category, similarity_strategy
-          @identifier    = "#{category.identifier}:#{name}"
-          @files         = Internals::Index::Files.new name, category
-
-          @index         = {}
-          @weights       = {}
-          @similarity    = {}
-          @configuration = {} # A hash with config options.
-
-          @similarity_strategy = similarity_strategy
-        end
-
-        # Get a list of similar texts.
-        #
-        # Note: Does not return itself.
-        #
-        def similar text
-          code = similarity_strategy.encoded text
-          similar_codes = code && @similarity[code]
-          similar_codes.delete text if similar_codes
-          similar_codes || []
-        end
-
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/indexing/category.rb

@@ -1,153 +0,0 @@
-module Internals
-
-  module Indexing
-
-    class Category
-
-      include Internals::Shared::Category
-
-      attr_reader :name, :index, :exact, :partial
-
-      # Mandatory params:
-      #  * name: Category name to use as identifier and file names.
-      #  * index: Index to which this category is attached to.
-      #
-      # Options:
-      #  * partial: Partial::None.new, Partial::Substring.new(from:start_char, to:up_to_char) (defaults from:-3, to:-1)
-      #  * similarity: Similarity::None.new (default), Similarity::DoubleMetaphone.new(amount_of_similarly_linked_words)
-      #  * from: The source category identifier to take the data from.
-      #
-      # Advanced Options:
-      #  * source: Use if the category should use a different source.
-      #  * weights: Query::Weights.new( [:category1, :category2] => +2, ... )
-      #  * tokenizer: Use a subclass of Tokenizers::Base that implements #tokens_for and #empty_tokens.
-      #  * key_format: What this category's keys are formatted with (default is :to_i)
-      #
-      def initialize name, index, options = {}
-        @name  = name
-        @index = index
-
-        @source     = options[:source]
-        @from       = options[:from]
-        @tokenizer  = options[:tokenizer]
-        @key_format = options[:key_format]
-
-        # TODO Push into Bundle. At least the weights.
-        #
-        partial    = options[:partial]    || Generators::Partial::Default
-        weights    = options[:weights]    || Generators::Weights::Default
-        similarity = options[:similarity] || Generators::Similarity::Default
-
-        bundle_class = index.bundle_class || Bundle::Memory
-
-        @exact   = bundle_class.new(:exact,   self, similarity, Generators::Partial::None.new, weights)
-        @partial = bundle_class.new(:partial, self, Generators::Similarity::None.new, partial, weights)
-      end
-
-      # Return an appropriate source.
-      #
-      def source
-        @source || @index.source
-      end
-      # Return the key format.
-      #
-      # If the source has no key format, then
-      # check for an explicit key format, and
-      # if none is defined, ask the index for
-      # one.
-      #
-      def key_format
-        source.respond_to?(:key_format) && source.key_format || @key_format || index.key_format
-      end
-      # The indexer is lazily generated and cached.
-      #
-      def indexer
-        @indexer ||= source.respond_to?(:each) ? Indexers::Parallel.new(self) : Indexers::Serial.new(self)
-      end
-      # TODO This is a hack to get the parallel indexer working. 
-      #
-      def categories
-        [self]
-      end
-      # Returns an appropriate tokenizer.
-      # If one isn't set on this category, will try the index,
-      # and finally the default index tokenizer.
-      #
-      def tokenizer
-        @tokenizer || @index.tokenizer || Tokenizers::Index.default
-      end
-
-      # Where the data is taken from.
-      #
-      def from
-        @from || name
-      end
-
-      def backup_caches
-        timed_exclaim "Backing up #{identifier}."
-        exact.backup
-        partial.backup
-      end
-      def restore_caches
-        timed_exclaim "Restoring #{identifier}."
-        exact.restore
-        partial.restore
-      end
-      def check_caches
-        timed_exclaim "Checking #{identifier}."
-        exact.raise_unless_cache_exists
-        partial.raise_unless_cache_exists
-      end
-      def clear_caches
-        timed_exclaim "Deleting #{identifier}."
-        exact.delete
-        partial.delete
-      end
-      
-      # Indexes, creates the "prepared_..." file.
-      #
-      def index!
-        prepare_index_directory
-        indexer.index
-      end
-
-      # Generates all caches for this category.
-      #
-      def cache!
-        prepare_index_directory
-        generate_caches
-      end
-      # We need to set what formatting method should be used.
-      # Uses the one defined in the indexer.
-      #
-      def configure
-        exact[:key_format] = self.key_format
-        partial[:key_format] = self.key_format
-      end
-      def generate_caches
-        configure
-        generate_caches_from_source
-        generate_partial
-        generate_caches_from_memory
-        dump_caches
-        timed_exclaim %Q{"#{identifier}": Caching finished.}
-      end
-      def generate_caches_from_source
-        exact.generate_caches_from_source
-      end
-      def generate_partial
-        partial.generate_partial_from exact.index
-      end
-      def generate_caches_from_memory
-        partial.generate_caches_from_memory
-      end
-      def dump_caches
-        exact.dump
-        partial.dump
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/indexing/index.rb

@@ -1,142 +0,0 @@
-# TODO Move to the API.
-#
-module Internals
-
-  module Indexing
-
-    class Index
-
-      attr_reader :name, :categories, :after_indexing, :bundle_class, :tokenizer
-
-      # Delegators for indexing.
-      #
-      delegate :connect_backend,
-               :to => :source
-
-      each_delegate :backup_caches,
-                    :cache!,
-                    :check_caches,
-                    :clear_caches,
-                    :create_directory_structure,
-                    :generate_caches,
-                    :restore_caches,
-                    :to => :categories
-
-      def initialize name, options = {}
-        @name           = name
-        @source         = options[:source]
-        @after_indexing = options[:after_indexing]
-        @bundle_class   = options[:indexing_bundle_class] # TODO This should actually be a fixed parameter.
-        @tokenizer      = options[:tokenizer]
-        @key_format     = options[:key_format]
-
-        @categories = []
-      end
-
-      # TODO Spec. Doc.
-      #
-      def define_category category_name, options = {}
-        new_category = Category.new category_name, self, options
-        new_category = yield new_category if block_given?
-        categories << new_category
-        new_category
-      end
-
-      # TODO Spec. Doc.
-      #
-      def define_indexing options = {}
-        @tokenizer = Internals::Tokenizers::Index.new options
-      end
-
-      #
-      #
-      def define_source source
-        @source = source
-      end
-      def source
-        @source || raise_no_source
-      end
-      def raise_no_source
-        raise NoSourceSpecifiedException.new(<<-NO_SOURCE
-
-
-No source given for index #{name}. An index needs a source.
-Example:
-  Index::Memory.new(:with_source) do
-    source   Sources::CSV.new(:title, file: 'data/books.csv')
-    category :title
-    category :author
-  end
-
-        NO_SOURCE
-)
-      end
-
-      #
-      #
-      def define_key_format key_format
-        @key_format = key_format
-      end
-      def key_format
-        @key_format || :to_i
-      end
-
-      #
-      #
-      def find category_name
-        category_name = category_name.to_sym
-
-        categories.each do |category|
-          next unless category.name == category_name
-          return category
-        end
-
-        raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
-      end
-
-      # Decides whether to use a parallel indexer or whether to
-      # delegate to each category to index themselves.
-      #
-      def index!
-        # TODO Duplicated in category.rb def indexer.
-        #
-        if source.respond_to?(:each)
-          warn %Q{\n\033[1mWarning\033[m, source for index "#{name}" is empty: #{source} (responds true to empty?).\n} if source.respond_to?(:empty?) && source.empty?
-          index_parallel
-        else
-          categories.each &:index!
-        end
-      end
-      # Indexes the categories in parallel.
-      #
-      # Only use where the category does not have a non-#each source defined.
-      #
-      def index_parallel
-        indexer = Indexers::Parallel.new self
-        categories.first.prepare_index_directory # TODO Unnice.
-        indexer.index
-      end
-
-      # Indexing.
-      #
-      # Note: If it is an each source we do not take a snapshot.
-      #
-      def take_snapshot
-        source.take_snapshot self unless source.respond_to? :each
-      end
-
-      #
-      #
-      def to_s
-        <<-INDEX
-Indexing(#{name}):
-#{"source: #{source}".indented_to_s}
-#{"Categories:\n#{categories.indented_to_s}".indented_to_s}
-INDEX
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/indexing/wrappers/category/location.rb

@@ -1,27 +0,0 @@
-module Internals
-  module Indexing
-    module Wrappers
-      module Category
-
-        module Location
-
-          def self.install_on category, grid, precision = 1
-            new_source = Sources::Wrappers::Location.new category.source, grid, precision
-
-            category.class_eval do
-              def tokenizer
-                @tokenizer ||= Internals::Tokenizers::Index.new
-              end
-              define_method :source do
-                new_source
-              end
-            end
-
-          end
-
-        end
-
-      end
-    end
-  end
-end

data/lib/picky/internals/query/allocation.rb

@@ -1,88 +0,0 @@
-module Internals
-
-  module Query
-
-    # An allocation has a number of combinations:
-    # [token, index] [other_token, other_index], ...
-    #
-    class Allocation # :nodoc:all
-
-      attr_reader :count, :ids, :score, :combinations, :result_identifier
-    
-      #
-      #
-      def initialize combinations, result_identifier
-        @combinations      = combinations
-        @result_identifier = result_identifier
-      end
-
-      def hash
-        @combinations.hash
-      end
-      def eql? other_allocation
-        true # FIXME
-        # @combinations.eql? other_allocation.combinations
-      end
-
-      # Scores its combinations and caches the result.
-      #
-      def calculate_score weights
-        @score ||= @combinations.calculate_score(weights)
-      end
-
-      # Asks the combinations for the (intersected) ids.
-      #
-      def calculate_ids amount, offset
-        @combinations.ids amount, offset # Calculate as many ids as are necessary.
-      end
-
-      # This starts the searching process.
-      #
-      def process! amount, offset
-        ids    = calculate_ids amount, offset
-        @count = ids.size                         # cache the count before throwing away the ids
-        @ids   = ids.slice!(offset, amount) || [] # slice out the relevant part
-      end
-
-      #
-      #
-      def keep identifiers = [] # categories
-        @combinations.keep identifiers
-      end
-      #
-      #
-      def remove identifiers = [] # categories
-        @combinations.remove identifiers
-      end
-
-      # Sort highest score first.
-      #
-      def <=> other_allocation
-         other_allocation.score <=> self.score
-      end
-
-      # Transform the allocation into result form.
-      #
-      def to_result
-        [self.result_identifier, self.score, self.count, @combinations.to_result, self.ids] if self.count > 0
-      end
-
-      # Json representation of this allocation.
-      #
-      # Note: Delegates to to_result.
-      #
-      def to_json
-        to_result.to_json
-      end
-
-      #
-      #
-      def to_s
-        "Allocation: #{to_result.join(', ')}"
-      end
-
-    end
-    
-  end
-  
-end

data/lib/picky/internals/query/allocations.rb

@@ -1,118 +0,0 @@
-module Internals
-
-  module Query
-    # Container class for allocations.
-    #
-    class Allocations # :nodoc:all
-
-      delegate :each, :inject, :empty?, :size, :to => :@allocations
-      attr_reader :total
-
-      def initialize allocations = []
-        @allocations = allocations
-      end
-
-      # Score each allocation.
-      #
-      def calculate_score weights
-        @allocations.each do |allocation|
-          allocation.calculate_score weights
-        end
-      end
-      # Sort the allocations.
-      #
-      def sort!
-        @allocations.sort!
-      end
-
-      # Reduces the amount of allocations to x.
-      #
-      def reduce_to amount
-        @allocations = @allocations.shift amount
-      end
-
-      # Keeps combinations.
-      #
-      # Only those passed in remain.
-      #
-      def keep identifiers = []
-        @allocations.each { |allocation| allocation.keep identifiers } unless identifiers.empty?
-      end
-      # Removes combinations.
-      #
-      # Only those passed in are removed.
-      #
-      def remove identifiers = []
-        @allocations.each { |allocation| allocation.remove identifiers } unless identifiers.empty?
-      end
-
-      # Returns the top amount ids.
-      #
-      def ids amount = 20
-        @allocations.inject([]) do |total, allocation|
-          total.size >= amount ? (return total.shift(amount)) : total + allocation.ids
-        end
-      end
-
-      # This is the main method of this class that will replace ids and count.
-      #
-      # What it does is calculate the ids and counts of its allocations
-      # for being used in the results. It also calculates the total
-      #
-      # Parameters:
-      #  * amount: the amount of ids to calculate
-      #  * offset: the offset from where in the result set to take the ids
-      #
-      # Note: With an amount of 0, an offset > 0 doesn't make much
-      #       sense, as seen in the live search.
-      #
-      # Note: Each allocation caches its count, but not its ids (thrown away).
-      #       The ids are cached in this class.
-      #
-      # Note: It's possible that no ids are returned by an allocation, but a count. (In case of an offset)
-      #
-      def process! amount, offset = 0
-        @total = 0
-        current_offset = 0
-        @allocations.each do |allocation|
-          ids = allocation.process! amount, offset
-          @total = @total + allocation.count # the total mixed in
-          if ids.empty?
-            offset = offset - allocation.count unless offset.zero?
-          else
-            amount = amount - ids.size # we need less results from the following allocation
-            offset = 0                 # we have already passed the offset
-          end
-        end
-      end
-
-      def uniq
-        @allocations.uniq!
-      end
-
-      def to_a
-        @allocations
-      end
-
-      # Simply inspects the internal allocations.
-      #
-      def to_s
-        @allocations.inspect
-      end
-
-      # Allocations for results are in the form:
-      # [
-      #   allocation1.to_result,
-      #   allocation2.to_result
-      #   ...
-      # ]
-      #
-      def to_result
-        @allocations.map(&:to_result).compact
-      end
-
-    end
-
-  end
-
-end

data/lib/picky/internals/query/combination.rb

@@ -1,80 +0,0 @@
-module Internals
-
-  module Query
-
-    # Describes the combination of a token (the text) and
-    # the index (the bundle): [text, index_bundle]
-    #
-    # A combination is a single part of an allocation:
-    # [..., [text2, index_bundle2], ...]
-    #
-    # An allocation consists of a number of combinations:
-    # [[text1, index_bundle1], [text2, index_bundle2], [text3, index_bundle1]]
-    #
-    class Combination # :nodoc:all
-
-      attr_reader :token, :bundle, :category_name
-
-      def initialize token, category
-        @token         = token
-        @category_name = category.name
-        @bundle        = category.bundle_for token
-        @text          = @token.text # don't want to use reset_similar already
-      end
-    
-      # Note: Required for uniq!
-      #
-      def hash
-        [@token.to_s, @bundle].hash
-      end
-    
-      # Returns the weight of this combination.
-      #
-      # Note: Caching is most of the time useful.
-      #
-      def weight
-        @weight ||= @bundle.weight(@text)
-      end
-    
-      # Returns an array of ids for the given text.
-      #
-      # Note: Caching is most of the time useful.
-      #
-      def ids
-        @ids ||= @bundle.ids(@text)
-      end
-    
-      # The identifier for this combination.
-      #
-      def identifier
-        "#{bundle.identifier}:#{@token.identifier}"
-      end
-    
-      # Is the identifier in the given identifiers?
-      #
-      def in? identifiers
-        identifiers.include? identifier
-      end
-
-      # Combines the category names with the original names.
-      # [
-      #  [:title,    'Flarbl', :flarbl],
-      #  [:category, 'Gnorf',  :gnorf]
-      # ]
-      #
-      def to_result
-        [@category_name, *@token.to_result]
-      end
-    
-      # Example:
-      #  "exact title:Peter*:peter"
-      #
-      def to_s
-        "#{bundle.identifier} #{to_result.join(':')}"
-      end
-    
-    end
-
-  end
-  
-end