picky 2.1.2 → 2.2.0

data/lib/picky/internals/indexing/category.rb

@@ -4,10 +4,9 @@ module Internals
 
     class Category
 
-      attr_reader :exact, :partial, :name, :configuration, :indexer
+      include Internals::Shared::Category
 
-      delegate :identifier, :prepare_index_directory, :to => :configuration
-      delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
+      attr_reader :name, :index, :exact, :partial
 
       # Mandatory params:
       #  * name: Category name to use as identifier and file names.
@@ -16,26 +15,20 @@ module Internals
       # Options:
       #  * partial: Partial::None.new, Partial::Substring.new(from:start_char, to:up_to_char) (defaults from:-3, to:-1)
       #  * similarity: Similarity::None.new (default), Similarity::DoubleMetaphone.new(amount_of_similarly_linked_words)
-      #  * source: Use if the category should use a different source.
       #  * from: The source category identifier to take the data from.
       #
       # Advanced Options:
-      #
+      #  * source: Use if the category should use a different source.
       #  * weights: Query::Weights.new( [:category1, :category2] => +2, ... )
       #  * tokenizer: Use a subclass of Tokenizers::Base that implements #tokens_for and #empty_tokens.
       #
-      # TODO Should source be not optional, or taken from the index?
-      #
       def initialize name, index, options = {}
-        @name = name
-        @from = options[:from]
-
-        # Now we have enough info to combine the index and the category.
-        #
-        @configuration = Configuration::Index.new index, self
+        @name  = name
+        @index = index
 
-        @tokenizer = options[:tokenizer] || Tokenizers::Index.default
-        @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer
+        @source    = options[:source]
+        @from      = options[:from]
+        @tokenizer = options[:tokenizer]
 
         # TODO Push into Bundle. At least the weights.
         #
@@ -44,20 +37,30 @@ module Internals
         similarity = options[:similarity] || Generators::Similarity::Default
 
         bundle_class = options[:indexing_bundle_class] || Bundle::Memory
-        @exact   = bundle_class.new(:exact,   configuration, similarity, Generators::Partial::None.new, weights)
-        @partial = bundle_class.new(:partial, configuration, Generators::Similarity::None.new, partial, weights)
+        @exact   = bundle_class.new(:exact,   self, similarity, Generators::Partial::None.new, weights)
+        @partial = bundle_class.new(:partial, self, Generators::Similarity::None.new, partial, weights)
       end
 
-      def to_s
-        <<-CATEGORY
-Category(#{name} from #{from}):
-  Exact:
-#{exact.indented_to_s(4)}
-  Partial:
-#{partial.indented_to_s(4)}
-        CATEGORY
+      # Return an appropriate source.
+      #
+      def source
+        @source || @index.source
+      end
+      # The indexer is lazily generated and cached.
+      #
+      def indexer
+        @indexer ||= Indexers::Serial.new self
+      end
+      # Returns an appropriate tokenizer.
+      # If one isn't set on this category, will try the index,
+      # and finally the default index tokenizer.
+      #
+      def tokenizer
+        @tokenizer || @index.tokenizer || Tokenizers::Index.default
       end
 
+      # Where the data is taken from.
+      #
       def from
         @from || name
       end
@@ -83,14 +86,14 @@ Category(#{name} from #{from}):
         partial.delete
       end
 
-      def index
+      def index!
         prepare_index_directory
         indexer.index
       end
 
       # Generates all caches for this category.
       #
-      def cache
+      def cache!
         prepare_index_directory
         configure
         generate_caches

data/lib/picky/internals/indexing/index.rb

@@ -6,60 +6,115 @@ module Internals
 
     class Index
 
-      attr_reader :name, :source, :categories, :after_indexing
+      attr_reader :name, :categories, :after_indexing, :bundle_class, :tokenizer
 
       # Delegators for indexing.
       #
       delegate :connect_backend,
                :to => :source
 
-      delegate :index,
-               :cache,
-               :generate_caches,
-               :backup_caches,
-               :restore_caches,
-               :check_caches,
-               :clear_caches,
-               :create_directory_structure,
-               :to => :categories
-
-      def initialize name, source, options = {}
-        @name   = name
-        @source = source
-
+      each_delegate :backup_caches,
+                    :cache!,
+                    :check_caches,
+                    :clear_caches,
+                    :create_directory_structure,
+                    :generate_caches,
+                    :restore_caches,
+                    :to => :categories
+
+      def initialize name, options = {}
+        @name           = name
+        @source         = options[:source]
         @after_indexing = options[:after_indexing]
         @bundle_class   = options[:indexing_bundle_class] # TODO This should actually be a fixed parameter.
+        @tokenizer      = options[:tokenizer]
 
-        @categories = Categories.new
+        @categories = []
       end
 
       # TODO Spec. Doc.
       #
       def define_category category_name, options = {}
-        options = default_category_options.merge options
-
         new_category = Category.new category_name, self, options
+        new_category = yield new_category if block_given?
         categories << new_category
         new_category
       end
 
-      # By default, the category uses
-      # * the index's source.
-      # * the index's bundle type.
+      # TODO Spec. Doc.
+      #
+      def define_indexing options = {}
+        @tokenizer = Internals::Tokenizers::Index.new options
+      end
+
+      #
+      #
+      def define_source source
+        @source = source
+      end
+      def source
+        @source || raise_no_source
+      end
+      def raise_no_source
+        raise NoSourceSpecifiedException.new(<<-NO_SOURCE
+
+
+No source given for index #{name}. An index needs a source.
+Example:
+  Index::Memory.new(:with_source) do
+    source   Sources::CSV.new(:title, file: 'data/books.csv')
+    category :title
+    category :author
+  end
+
+        NO_SOURCE
+)
+      end
+
+      #
       #
-      def default_category_options
-        {
-          :source => @source,
-          :indexing_bundle_class => @bundle_class
-        }
+      def find category_name
+        category_name = category_name.to_sym
+
+        categories.each do |category|
+          next unless category.name == category_name
+          return category
+        end
+
+        raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
+      end
+
+      # Decides whether to use a parallel indexer or whether to
+      # delegate to each category to index themselves.
+      #
+      def index!
+        if source.respond_to?(:each)
+          warn %Q{\n\033[1mWarning\033[m, source for index "#{name}" is empty: #{source} (responds true to empty?).\n} if source.respond_to?(:empty?) && source.empty?
+          index_parallel
+        else
+          categories.each &:index!
+        end
+      end
+      # Indexes the categories in parallel.
+      #
+      # Only use where the category does not have a non-#each source defined.
+      #
+      def index_parallel
+        indexer = Indexers::Parallel.new self
+        categories.first.prepare_index_directory # TODO Unnice.
+        indexer.index
       end
 
       # Indexing.
       #
+      # Note: If it is an each source we do not take a snapshot.
+      #
       def take_snapshot
-        source.take_snapshot self
+        source.take_snapshot self unless source.respond_to? :each
       end
 
+      #
+      #
       def to_s
         <<-INDEX
 Indexing(#{name}):

data/lib/picky/internals/indexing/wrappers/category/location.rb

@@ -0,0 +1,27 @@
+module Internals
+  module Indexing
+    module Wrappers
+      module Category
+
+        module Location
+
+          def self.install_on category, grid, precision = 1
+            new_source = Sources::Wrappers::Location.new category.source, grid, precision
+
+            category.class_eval do
+              def tokenizer
+                @tokenizer ||= Internals::Tokenizers::Index.new
+              end
+              define_method :source do
+                new_source
+              end
+            end
+
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/picky/internals/query/indexes.rb

@@ -22,7 +22,7 @@ module Internals
       #
       def initialize *index_definitions, combinations_type
         @combinations_type = combinations_type
-        @indexes           = index_definitions.map &:indexed
+        @indexes           = index_definitions.map(&:internal_indexed)
       end
 
       # Returns a number of prepared (sorted, reduced etc.) allocations for the given tokens.

data/lib/picky/internals/query/qualifiers.rb

@@ -35,8 +35,6 @@ module Internals
     #
     class Qualifiers # :nodoc:all
 
-      include Singleton
-
       attr_reader :qualifiers, :normalization_mapping
 
       delegate :<<, :to => :qualifiers
@@ -47,13 +45,16 @@ module Internals
         @qualifiers = []
         @normalization_mapping = {}
       end
-    
+      def self.instance
+        @instanec ||= new
+      end
+
       # TODO Spec.
       #
       def self.add name, qualifiers
         instance << Qualifier.new(name, qualifiers)
       end
-    
+
       # Uses the qualifiers to prepare (optimize) the qualifier handling.
       #
       def prepare
@@ -75,7 +76,7 @@ module Internals
       end
 
     end
-    
+
   end
-  
+
 end

data/lib/picky/internals/query/weights.rb

@@ -4,6 +4,8 @@ module Query
   #
   class Weights # :nodoc:all
 
+    attr_reader :weights
+
     #
     #
     def initialize weights = {}
@@ -46,6 +48,10 @@ module Query
       @weights.empty?
     end
 
+    def == other
+      @weights == other.weights
+    end
+
     # Prints out a nice representation of the configured weights.
     #
     def to_s

data/lib/picky/internals/shared/category.rb

@@ -0,0 +1,52 @@
+module Internals
+  module Shared
+
+    module Category
+
+      def index_name
+        index.name
+      end
+      def category_name
+        name
+      end
+
+      # Path and partial filename of a specific index on this category.
+      #
+      def index_path bundle_name, type
+        "#{index_directory}/#{name}_#{bundle_name}_#{type}"
+      end
+
+      #
+      #
+      def prepared_index_path
+        @prepared_index_path ||= "#{index_directory}/prepared_#{name}_index"
+      end
+      def prepared_index_file &block
+        @prepared_index_file ||= Internals::Index::File::Text.new prepared_index_path
+        @prepared_index_file.open_for_indexing &block
+      end
+
+      # Identifier for internal use.
+      #
+      def identifier
+        @identifier ||= "#{index.name}:#{name}"
+      end
+      def to_s
+        "#{index.name} #{name}"
+      end
+
+      # The index directory for this category.
+      #
+      def index_directory
+        @index_directory ||= "#{PICKY_ROOT}/index/#{PICKY_ENVIRONMENT}/#{index.name}"
+      end
+      # Creates the index directory including all necessary paths above it.
+      #
+      def prepare_index_directory
+        FileUtils.mkdir_p index_directory
+      end
+
+    end
+
+  end
+end

data/lib/picky/internals/tokenizers/base.rb

@@ -169,7 +169,7 @@ Case sensitive?            #{@case_sensitive ? "Yes." : "-"}
         # Defaults.
         #
         splits_text_on options[:splits_text_on] || /\s/
-        reject_token_if &(options[:reject_token_if] || :blank?)
+        reject_token_if &(options[:reject_token_if] || options[:rejects_token_if] || :blank?) # TODO Decide on using an s or not.
       end
 
       # Default preprocessing hook.

data/lib/picky/internals/tokenizers/location.rb

@@ -0,0 +1,54 @@
+module Internals
+
+  module Tokenizers
+
+
+    class Location < Base
+
+      attr_reader :calculation
+
+      def initialize options = {}
+        super options
+
+        grid      = options[:grid]
+        precision = options[:precision] || 1
+
+        @calculation = Internals::Calculations::Location.new grid, precision
+
+        @minimum = 1.0 / 0
+
+        @locations = []
+      end
+
+      # TODO Work on this!
+      #
+      def tokenize text
+
+        # Gather min/max.
+        #
+        source.harvest category do |indexed_id, location|
+          location = location.to_f
+          minimum = location if location < minimum
+          locations << [indexed_id, location]
+        end
+
+        calculation.minimum = minimum
+
+        # Recalculate locations.
+        #
+        locations.each do |indexed_id, location|
+          calculation.recalculated_range(location).each do |new_location|
+            yield indexed_id, new_location.to_s
+          end
+        end
+
+        # TODO Move to the right place.
+        #
+        category.exact[:location_minimum] = minimum
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/loader.rb

@@ -109,8 +109,9 @@ module Loader # :nodoc:all
 
     # Index generation strategies.
     #
-    load_internals 'indexers/no_source_specified_error'
+    load_internals 'indexers/base'
     load_internals 'indexers/serial'
+    load_internals 'indexers/parallel'
 
     # Generators.
     #
@@ -146,6 +147,10 @@ module Loader # :nodoc:all
     load_internals 'generators/weights_generator'
     load_internals 'generators/similarity_generator'
 
+    # Shared index elements.
+    #
+    load_internals 'shared/category'
+
     # Index store handling.
     #
     load_internals 'index/backend'
@@ -169,9 +174,11 @@ module Loader # :nodoc:all
     load_internals 'indexing/bundle/memory'
     load_internals 'indexing/bundle/redis'
     load_internals 'indexing/category'
-    load_internals 'indexing/categories'
+    # load_internals 'indexing/categories'
     load_internals 'indexing/index'
 
+    load_internals 'indexing/wrappers/category/location'
+
     load_internals 'indexed/bundle/base'
     load_internals 'indexed/bundle/memory'
     load_internals 'indexed/bundle/redis'
@@ -187,6 +194,8 @@ module Loader # :nodoc:all
     load_internals 'indexed/wrappers/bundle/calculation'
     load_internals 'indexed/wrappers/bundle/location'
 
+    load_internals 'indexed/wrappers/category/location'
+
     # Tokens.
     #
     load_internals 'query/token'
@@ -216,7 +225,7 @@ module Loader # :nodoc:all
 
     # Configuration.
     #
-    load_internals 'configuration/index'
+    # load_internals 'configuration/index'
 
     # Adapters.
     #
@@ -236,6 +245,10 @@ module Loader # :nodoc:all
     #
     load_relative 'rack/harakiri'
 
+    # Errors.
+    #
+    load_relative 'no_source_specified_exception'
+
     # Load analyzer.
     #
     load_relative 'analyzer'

data/lib/picky/no_source_specified_exception.rb

@@ -0,0 +1,3 @@
+# Raised if no source is available on a category.
+#
+class NoSourceSpecifiedException < StandardError; end

data/lib/picky/search.rb

@@ -25,13 +25,52 @@ class Search
   #
   # TODO Add identifiers_to_remove (rename) and reduce_allocations_to_amount (rename).
   #
+  # It is also possible to define the tokenizer and weights like so.
+  # Example:
+  #   Search.new(index1, index2, index3) do
+  #     searching removes_characters: /[^a-z]/, etc.
+  #     weights [:author, :title] => +3, [:title, :isbn] => +1
+  #   end
+  #
   def initialize *index_definitions
     options      = Hash === index_definitions.last ? index_definitions.pop : {}
 
-    @indexes     = Internals::Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
-    @tokenizer   = options[:tokenizer] || Internals::Tokenizers::Query.default
-    weights      = options[:weights] || Query::Weights.new
-    @weights     = Hash === weights ? Query::Weights.new(weights) : weights
+    @indexes  = Internals::Query::Indexes.new *index_definitions, combinations_type_for(index_definitions)
+    searching options[:tokenizer]
+    boost     options[:weights]
+
+    instance_eval(&Proc.new) if block_given?
+  end
+
+  # TODO Doc. Spec.
+  #
+  # Example:
+  #   Search.new(index1, index2, index3) do
+  #     searching removes_characters: /[^a-z]/, etc.
+  #     weights [:author, :title] => +3, [:title, :isbn] => +1
+  #   end
+  #
+  def searching options
+    @tokenizer = if options.respond_to?(:tokenize)
+      options
+    else
+      options && Internals::Tokenizers::Query.new(options)
+    end
+  end
+  def tokenizer
+    @tokenizer || Internals::Tokenizers::Query.default
+  end
+  # TODO Doc. Spec.
+  #
+  # Example:
+  #   Search.new(index1, index2, index3) do
+  #     searching removes_characters: /[^a-z]/, etc.
+  #     boost [:author, :title] => +3, [:title, :isbn] => +1
+  #   end
+  #
+  def boost options
+    weights  = options || Query::Weights.new
+    @weights = Hash === weights ? Query::Weights.new(weights) : weights
   end
 
   # Returns the right combinations strategy for
@@ -110,7 +149,7 @@ class Search
   # * text: The text to tokenize.
   #
   def tokenized text
-    @tokenizer.tokenize text
+    tokenizer.tokenize text
   end
 
   # Gets sorted allocations for the tokens.

data/lib/picky/sources/base.rb

@@ -42,7 +42,7 @@ module Sources
 
     # Called by the indexer when gathering data.
     #
-    # Yields the data (id, text for id) for the given type and category.
+    # Yields the data (id, text for id) for the given category.
     #
     # When implementing or overriding your own,
     # be sure to <tt>yield(id, text_for_id)</tt> (or <tt>block.call(id, text_for_id)</tt>)
@@ -50,7 +50,7 @@ module Sources
     #
     # Note: Since harvest needs to be implemented, it has no default impementation.
     #
-    # def harvest index, category # :yields: id, text_for_id
+    # def harvest category # :yields: id, text_for_id
     #
     # end
 

data/lib/picky/sources/couch.rb

@@ -51,7 +51,7 @@ module Sources
     # See important note, above.
     #
     @@id_key = '_id'
-    def harvest type, category
+    def harvest category
       category_name = category.from.to_s
       get_data do |doc|
         yield doc[@@id_key], doc[category_name] || next

data/lib/picky/sources/csv.rb

@@ -58,7 +58,7 @@ module Sources
 
     # Harvests the data to index.
     #
-    def harvest _, category
+    def harvest category
       index = category_names.index category.from
       get_data do |ary|
         indexed_id = ary.shift

data/lib/picky/sources/db.rb

@@ -123,19 +123,19 @@ module Sources
 
     # Harvests the data to index in chunks.
     #
-    def harvest index, category, &block
+    def harvest category, &block
       connect_backend
 
-      (0..count(index)).step(chunksize) do |offset|
-        get_data index, category, offset, &block
+      (0..count(category.index)).step(chunksize) do |offset|
+        get_data category, offset, &block
       end
     end
 
     # Gets the data from the backend.
     #
-    def get_data index, category, offset, &block # :nodoc:
+    def get_data category, offset, &block # :nodoc:
 
-      select_statement = harvest_statement_with_offset index, category, offset
+      select_statement = harvest_statement_with_offset category, offset
 
       # TODO Rewrite ASAP.
       #
@@ -155,8 +155,8 @@ module Sources
 
     # Builds a harvest statement for getting data to index.
     #
-    def harvest_statement_with_offset index, category, offset
-      statement = harvest_statement index, category
+    def harvest_statement_with_offset category, offset
+      statement = harvest_statement category
 
       statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
 
@@ -165,8 +165,8 @@ module Sources
 
     # The harvest statement used to pull data from the snapshot table.
     #
-    def harvest_statement index, category
-      "SELECT id, #{category.from} FROM #{snapshot_table_name(index)} st"
+    def harvest_statement category
+      "SELECT id, #{category.from} FROM #{snapshot_table_name(category.index)} st"
     end
 
     # The amount of records that are loaded each chunk.