picky 2.5.2 → 2.6.0

data/lib/picky/index/base.rb

@@ -85,7 +85,11 @@ module Index
   #
   class Base
 
-    attr_reader :name
+    attr_reader :name,
+                :categories
+
+    delegate :[],
+             :to => :categories
 
     # Create a new index with a given source.
     #
@@ -115,8 +119,22 @@ module Index
       @name = name.to_sym
 
       check_options options
-      @indexing = Internals::Indexing::Index.new name, options
-      @indexed  = Internals::Indexed::Index.new  name, options
+
+      @source = options[:source]
+
+      @after_indexing        = options[:after_indexing]
+      @indexing_bundle_class = options[:indexing_bundle_class] # TODO This should probably be a fixed parameter.
+      @tokenizer             = options[:tokenizer]
+      @key_format            = options[:key_format]
+
+      # Indexed.
+      #
+      @result_identifier    = options[:result_identifier] || name
+      @indexed_bundle_class = options[:indexed_bundle_class] # TODO This should probably be a fixed parameter.
+
+      # TODO Move ignore_unassigned_tokens to query, somehow.
+      #
+      @categories = Categories.new ignore_unassigned_tokens: (options[:ignore_unassigned_tokens] || false)
 
       # Centralized registry.
       #
@@ -126,104 +144,19 @@ module Index
       #
       instance_eval(&Proc.new) if block_given?
 
-      check_source internal_indexing.source
-    end
-    def internal_indexing # :nodoc:
-      @indexing
-    end
-    def internal_indexed # :nodoc:
-      @indexed
-    end
-    #
-    # Since this is an API, we fail hard quickly.
-    #
-    def check_name name # :nodoc:
-      raise ArgumentError.new(<<-NAME
-
-
-The index identifier (you gave "#{name}") for Index::Memory/Index::Redis should be a Symbol/String,
-Examples:
-  Index::Memory.new(:my_cool_index) # Recommended
-  Index::Redis.new("a-redis-index")
-NAME
-
-
-) unless name.respond_to?(:to_sym)
-    end
-    def check_options options # :nodoc:
-      raise ArgumentError.new(<<-OPTIONS
-
-
-Sources are not passed in as second parameter for #{self.class.name} anymore, but either
-* as :source option:
-  #{self.class.name}.new(#{name.inspect}, source: #{options})
-or
-* given to the #source method inside the config block:
-  #{self.class.name}.new(#{name.inspect}) do
-    source #{options}
-  end
-
-Sorry about that breaking change (in 2.2.0), didn't want to go to 3.0.0 yet!
-
-All the best
-  -- Picky
-
-
-OPTIONS
-) unless options.respond_to?(:[])
-    end
-    def check_source source # :nodoc:
-      raise ArgumentError.new(<<-SOURCE
-
-
-The index "#{name}" should use a data source that responds to either the method #each, or the method #harvest, which yields(id, text).
-Or it could use one of the built-in sources:
-  Sources::#{(Sources.constants - [:Base, :Wrappers, :NoCSVFileGiven, :NoCouchDBGiven]).join(',
-  Sources::')}
-
-
-SOURCE
-) unless source.respond_to?(:each) || source.respond_to?(:harvest)
-    end
-
-    def to_stats # :nodoc:
-      stats = <<-INDEX
-#{name} (#{self.class}):
-  #{"source:            #{internal_indexing.source}".indented_to_s}
-  #{"categories:        #{internal_indexing.categories.map(&:name).join(', ')}".indented_to_s}
-INDEX
-      stats << "  result identifier: \"#{internal_indexed.result_identifier}\"".indented_to_s unless internal_indexed.result_identifier.to_s == internal_indexed.name.to_s
-      stats
-    end
-
-    # Define an index tokenizer on the index.
-    #
-    # Parameters are the exact same as for indexing.
-    #
-    def indexing options = {}
-      internal_indexing.define_indexing options
+      # Check if any source has been given in the block or the options.
+      #
+      check_source @source
     end
-    alias define_indexing indexing
 
-    # Define a source on the index.
-    #
-    # Parameter is a source, either one of the standard sources or
-    # anything responding to #each and returning objects that
-    # respond to id and the category names (or the category from option).
+    # Default bundles.
     #
-    def source source
-      internal_indexing.define_source source
+    def indexing_bundle_class
+      Indexing::Bundle::Memory
     end
-    alias define_source source
-
-    # Define a key_format on the index.
-    #
-    # Parameter is a method name to use on the key (e.g. :to_i, :to_s, :strip).
-    #
-    def key_format key_format
-      internal_indexing.define_key_format key_format
+    def indexed_bundle_class
+      Indexed::Bundle::Memory
     end
-    alias define_key_format key_format
 
     # Defines a searchable category on the index.
     #
@@ -239,17 +172,26 @@ INDEX
     # * from: Take the data from the data category with this name. Example: You have a source Sources::CSV.new(:title, file:'some_file.csv') but you want the category to be called differently. The you use from: define_category(:similar_title, :from => :title).
     #
     def category category_name, options = {}
-      category_name = category_name.to_sym
+      options = default_category_options.merge options
 
-      indexing_category = internal_indexing.define_category category_name, options
-      indexed_category  = internal_indexed.define_category  category_name, options
+      new_category = Category.new category_name.to_sym, self, options
+      categories << new_category
 
-      yield indexing_category, indexed_category if block_given?
+      new_category = yield new_category if block_given?
 
-      self
+      new_category
     end
     alias define_category category
 
+    # By default, the category uses
+    # * the index's bundle type.
+    #
+    def default_category_options
+      {
+        :indexed_bundle_class => @indexed_bundle_class
+      }
+    end
+
     # Make this category range searchable with a fixed range. If you need other
     # ranges, define another category with a different range value.
     #
@@ -318,9 +260,9 @@ INDEX
 
       options = { partial: Partial::None.new }.merge options
 
-      define_category category_name, options do |indexing_category, indexed_category|
-        Internals::Indexing::Wrappers::Category::Location.install_on indexing_category, range, precision
-        Internals::Indexed::Wrappers::Category::Location.install_on indexed_category, range, precision
+      define_category category_name, options do |category|
+        Indexing::Wrappers::Category::Location.install_on category, range, precision
+        Indexed::Wrappers::Category::Location.install_on category, range, precision
       end
     end
     alias define_ranged_category ranged_category
@@ -386,6 +328,79 @@ INDEX
 
     end
     alias define_geo_categories geo_categories
+
+    #
+    # Since this is an API, we fail hard quickly.
+    #
+    def check_name name # :nodoc:
+      raise ArgumentError.new(<<-NAME
+
+
+The index identifier (you gave "#{name}") for Index::Memory/Index::Redis should be a Symbol/String,
+Examples:
+  Index::Memory.new(:my_cool_index) # Recommended
+  Index::Redis.new("a-redis-index")
+NAME
+
+
+) unless name.respond_to?(:to_sym)
+    end
+    def check_options options # :nodoc:
+      raise ArgumentError.new(<<-OPTIONS
+
+
+Sources are not passed in as second parameter for #{self.class.name} anymore, but either
+* as :source option:
+  #{self.class.name}.new(#{name.inspect}, source: #{options})
+or
+* given to the #source method inside the config block:
+  #{self.class.name}.new(#{name.inspect}) do
+    source #{options}
+  end
+
+Sorry about that breaking change (in 2.2.0), didn't want to go to 3.0.0 yet!
+
+All the best
+  -- Picky
+
+
+OPTIONS
+) unless options.respond_to?(:[])
+    end
+    def check_source source # :nodoc:
+      raise ArgumentError.new(<<-SOURCE
+
+
+The index "#{name}" should use a data source that responds to either the method #each, or the method #harvest, which yields(id, text).
+Or it could use one of the built-in sources:
+  Sources::#{(Sources.constants - [:Base, :Wrappers, :NoCSVFileGiven, :NoCouchDBGiven]).join(',
+  Sources::')}
+
+
+SOURCE
+) unless source.respond_to?(:each) || source.respond_to?(:harvest)
+    end
+
+    def method_name
+
+    end
+
+    #
+    #
+    def to_s
+      "#{self.class}(#{name}, result_id: #{result_identifier}, source: #{source}, categories: #{categories})"
+    end
+
+    def to_stats # :nodoc:
+      stats = <<-INDEX
+#{name} (#{self.class}):
+#{"source:            #{source}".indented_to_s}
+#{"categories:        #{categories.map(&:name).join(', ')}".indented_to_s}
+INDEX
+      stats << "  result identifier: \"#{result_identifier}\"".indented_to_s unless result_identifier.to_s == name.to_s
+      stats
+    end
+
   end
 
 end

data/lib/picky/index/base_indexed.rb

@@ -0,0 +1,27 @@
+module Index
+
+  #
+  #
+  class Base
+
+    attr_reader :result_identifier,
+                :combinator
+
+    delegate :load_from_cache,
+             :analyze,
+             :reindex,
+             :to => :categories
+
+    alias reload load_from_cache
+
+    # Return the possible combinations for this token.
+    #
+    # A combination is a tuple <token, index_bundle>.
+    #
+    def possible_combinations token
+      categories.possible_combinations_for token
+    end
+
+  end
+
+end

data/lib/picky/index/base_indexing.rb

@@ -0,0 +1,119 @@
+module Index
+
+  #
+  #
+  class Base
+    
+    attr_reader :after_indexing,
+                :bundle_class,
+                :tokenizer
+    
+    # Delegators for indexing.
+    #
+    delegate :backup_caches,
+             :cache,
+             :check_caches,
+             :clear_caches,
+             :create_directory_structure,
+             :generate_caches,
+             :restore_caches,
+             :to => :categories
+    
+    delegate :connect_backend,
+             :to => :source
+    
+    # Calling index on an index will
+    #  * prepare (the data)
+    #  * cache (the data)
+    # on every category.
+    #
+    def index
+      prepare
+      cache
+    end
+
+    # Define an index tokenizer on the index.
+    #
+    # Parameters are the exact same as for indexing.
+    #
+    def indexing options = {}
+      @tokenizer = Tokenizers::Index.new options
+    end
+    alias define_indexing indexing
+    
+    # Define a source on the index.
+    #
+    # Parameter is a source, either one of the standard sources or
+    # anything responding to #each and returning objects that
+    # respond to id and the category names (or the category from option).
+    #
+    def source some_source = nil
+      some_source ? define_source(some_source) : (@source || raise_no_source)
+    end
+    def define_source source
+      @source = source
+    end
+    def raise_no_source
+      raise NoSourceSpecifiedException.new(<<-NO_SOURCE
+
+
+No source given for index #{name}. An index needs a source.
+Example:
+Index::Memory.new(:with_source) do
+  source   Sources::CSV.new(:title, file: 'data/books.csv')
+  category :title
+  category :author
+end
+
+      NO_SOURCE
+)
+    end
+    
+    # Define a key_format on the index.
+    #
+    # Parameter is a method name to use on the key (e.g. :to_i, :to_s, :strip).
+    #
+    def key_format format = nil
+      format ? define_key_format(format) : (@key_format || :to_i)
+    end
+    def define_key_format key_format
+      @key_format = key_format
+    end
+    
+    # Decides whether to use a parallel indexer or whether to
+    # delegate to each category to index themselves.
+    #
+    # TODO Rename to prepare.
+    #
+    def prepare
+      # TODO Duplicated in category.rb def indexer.
+      #
+      if source.respond_to?(:each)
+        warn %Q{\n\033[1mWarning\033[m, source for index "#{name}" is empty: #{source} (responds true to empty?).\n} if source.respond_to?(:empty?) && source.empty?
+        index_parallel
+      else
+        categories.each &:prepare
+      end
+    end
+
+    # Indexes the categories in parallel.
+    #
+    # Only use where the category does not have a non-#each source defined.
+    #
+    def index_parallel
+      indexer = Indexers::Parallel.new self
+      categories.first.prepare_index_directory # TODO Unnice.
+      indexer.index
+    end
+
+    # Indexing.
+    #
+    # Note: If it is an each source we do not take a snapshot.
+    #
+    def take_snapshot
+      source.take_snapshot self unless source.respond_to? :each
+    end
+    
+  end
+  
+end

data/lib/picky/index/memory.rb

@@ -3,24 +3,12 @@ module Index
   # An index that is persisted in files, loaded at startup and kept in memory at runtime.
   #
   class Memory < Base
-
-    # Create a new memory index for indexing and for querying.
-    #
-    # Parameters:
-    # * name: The identifier of the index. Used:
-    #   - to identify an index (e.g. by you in Rake tasks: Indexes[:the_identifier]).
-    #   - in the frontend to describe which index a result came from.
-    #   - index directory naming (index/development/the_identifier/<lots of indexes>)
-    # * source: The source the data comes from. See Sources::Base.
-    #
-    # Options:
-    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index.
-    #
-    def initialize name, options = {}
-      options[:indexing_bundle_class] ||= Internals::Indexing::Bundle::Memory
-      options[:indexed_bundle_class]  ||= Internals::Indexed::Bundle::Memory
-
-      super name, options
+    
+    def indexing_bundle_class
+      Indexing::Bundle::Memory
+    end
+    def indexed_bundle_class
+      Indexed::Bundle::Memory
     end
 
   end

data/lib/picky/index/redis.rb

@@ -3,24 +3,12 @@ module Index
   # An index that is persisted in Redis.
   #
   class Redis < Base
-
-    # Create a new Redis index for indexing and for querying.
-    #
-    # Parameters:
-    # * name: The identifier of the index. Used:
-    #   - to identify an index (e.g. by you in Rake tasks: Indexes[:the_identifier]).
-    #   - in the frontend to describe which index a result came from.
-    #   - index directory naming (index/development/the_identifier/<lots of indexes>)
-    # * source: The source the data comes from. See Sources::Base.
-    #
-    # Options:
-    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index.
-    #
-    def initialize name, options = {}
-      options[:indexing_bundle_class] ||= Internals::Indexing::Bundle::Redis
-      options[:indexed_bundle_class]  ||= Internals::Indexed::Bundle::Redis
-
-      super name, options
+    
+    def indexing_bundle_class
+      Indexing::Bundle::Redis
+    end
+    def indexed_bundle_class
+      Indexed::Bundle::Redis
     end
 
   end

data/lib/picky/indexed/bundle/base.rb

@@ -0,0 +1,110 @@
+module Indexed # :nodoc:all
+
+  # A Bundle is a number of indexes
+  # per [index, category] combination.
+  #
+  # At most, there are three indexes:
+  # * *core* index (always used)
+  # * *weights* index (always used)
+  # * *similarity* index (used with similarity)
+  #
+  # In Picky, indexing is separated from the index
+  # handling itself through a parallel structure.
+  #
+  # Both use methods provided by this base class, but
+  # have very different goals:
+  #
+  # * *Indexing*::*Bundle* is just concerned with creating index files
+  #   and providing helper functions to e.g. check the indexes.
+  #
+  # * *Index*::*Bundle* is concerned with loading these index files into
+  #   memory and looking up search data as fast as possible.
+  #
+  module Bundle
+
+    class Base
+
+      attr_reader   :identifier, :configuration
+      attr_accessor :similarity_strategy
+      attr_accessor :index, :weights, :similarity, :configuration
+
+      delegate :[], :to => :configuration
+      delegate :size, :to => :index
+
+      def initialize name, category, similarity_strategy
+        @identifier = "#{category.identifier}:#{name}"
+
+        @index         = {}
+        @weights       = {}
+        @similarity    = {}
+
+        @similarity_strategy = similarity_strategy
+      end
+
+      # Get a list of similar texts.
+      #
+      # Note: Does not return itself.
+      #
+      def similar text
+        code = similarity_strategy.encoded text
+        similar_codes = code && @similarity[code]
+        similar_codes.delete text if similar_codes
+        similar_codes || []
+      end
+
+      # Loads all indexes.
+      #
+      def load
+        load_index
+        load_weights
+        load_similarity
+        load_configuration
+      end
+
+      # Loads the core index.
+      #
+      def load_index
+        # No loading needed.
+      end
+      # Loads the weights index.
+      #
+      def load_weights
+        # No loading needed.
+      end
+      # Loads the similarity index.
+      #
+      def load_similarity
+        # No loading needed.
+      end
+      # Loads the configuration.
+      #
+      def load_configuration
+        # No loading needed.
+      end
+
+      # Loads the core index.
+      #
+      def clear_index
+        # No loading needed.
+      end
+      # Loads the weights index.
+      #
+      def clear_weights
+        # No loading needed.
+      end
+      # Loads the similarity index.
+      #
+      def clear_similarity
+        # No loading needed.
+      end
+      # Loads the configuration.
+      #
+      def clear_configuration
+        # No loading needed.
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexed/bundle/memory.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+#
+module Indexed # :nodoc:all
+
+  #
+  #
+  module Bundle
+
+    # This is the _actual_ index (based on memory).
+    #
+    # Handles exact/partial index, weights index, and similarity index.
+    #
+    # Delegates file handling and checking to an *Indexed*::*Files* object.
+    #
+    class Memory < Base
+
+      delegate :[], :to => :configuration
+
+      def initialize name, configuration, *args
+        super name, configuration, *args
+
+        @configuration = {} # A hash with config options.
+
+        @backend = Backend::Files.new name, configuration
+      end
+
+      def to_s
+        <<-MEMORY
+Memory
+#{@backend.indented_to_s}
+MEMORY
+      end
+
+      # Get the ids for the given symbol.
+      #
+      def ids sym
+        @index[sym] || []
+      end
+      # Get a weight for the given symbol.
+      #
+      def weight sym
+        @weights[sym]
+      end
+
+      # Loads the core index.
+      #
+      def load_index
+        self.index = @backend.load_index
+      end
+      # Loads the weights index.
+      #
+      def load_weights
+        self.weights = @backend.load_weights
+      end
+      # Loads the similarity index.
+      #
+      def load_similarity
+        self.similarity = @backend.load_similarity
+      end
+      # Loads the configuration.
+      #
+      def load_configuration
+        self.configuration = @backend.load_configuration
+      end
+
+      # Loads the core index.
+      #
+      def clear_index
+        self.index = {}
+      end
+      # Loads the weights index.
+      #
+      def clear_weights
+        self.weights = {}
+      end
+      # Loads the similarity index.
+      #
+      def clear_similarity
+        self.similarity = {}
+      end
+      # Loads the configuration.
+      #
+      def clear_configuration
+        self.configuration = {}
+      end
+
+    end
+
+  end
+
+end