picky 0.11.2 → 0.12.0

data/lib/picky/indexed/bundle.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+#
+module Indexed
+
+  # This is the _actual_ index.
+  #
+  # Handles exact/partial index, weights index, and similarity index.
+  #
+  # Delegates file handling and checking to an *Indexed*::*Files* object.
+  #
+  class Bundle < Index::Bundle
+    
+    # Get the ids for the given symbol.
+    #
+    def ids sym
+      @index[sym] || []
+    end
+    # Get a weight for the given symbol.
+    #
+    def weight sym
+      @weights[sym]
+    end
+    
+    # Load the data from the db.
+    #
+    def load_from_index_file
+      load_from_index_generation_message
+      clear
+      retrieve
+    end
+    # Notifies the user that the index is being loaded.
+    #
+    def load_from_index_generation_message
+      timed_exclaim "LOAD INDEX #{identifier}."
+    end
+    
+    # Loads all indexes.
+    #
+    def load
+      load_index
+      load_similarity
+      load_weights
+    end
+    # Loads the core index.
+    #
+    def load_index
+      self.index = files.load_index
+    end
+    # Loads the weights index.
+    #
+    def load_weights
+      self.weights = files.load_weights
+    end
+    # Loads the similarity index.
+    #
+    def load_similarity
+      self.similarity = files.load_similarity
+    end
+    
+  end
+end

data/lib/picky/{index → indexed}/categories.rb

@@ -1,4 +1,4 @@
-module Index
+module Indexed
   
   class Categories
     

data/lib/picky/{index → indexed}/category.rb

@@ -1,38 +1,41 @@
-module Index
+module Indexed
   
-  # An index category holds a exact and a partial index for a given field.
+  # An index category holds a exact and a partial index for a given category.
   #
   # For example an index category for names holds a exact and
   # a partial index bundle for names.
   #
   class Category
     
-    attr_reader :name, :type, :exact, :partial
+    attr_reader :exact, :partial, :identifier, :name
     
     #
     #
-    def initialize name, type, options = {}
+    def initialize name, index, options = {}
       @name = name
-      @type = type
+      
+      configuration = Configuration::Index.new index, self
+      
+      @identifier    = configuration.identifier
       
       similarity = options[:similarity] || Cacher::Similarity::Default
       
-      @exact   = options[:exact_index_bundle]   || Bundle.new(:exact,   self, type, similarity)
-      @partial = options[:partial_index_bundle] || Bundle.new(:partial, self, type, similarity)
+      @exact   = options[:exact_index_bundle]   || Bundle.new(:exact,   configuration, similarity)
+      @partial = options[:partial_index_bundle] || Bundle.new(:partial, configuration, similarity)
       
       @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
       @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
       
       # Extract?
       #
-      qualifiers = generate_qualifiers_from options
-      Query::Qualifiers.add(name, qualifiers) if qualifiers
+      qualifiers = generate_qualifiers_from options || [name]
+      Query::Qualifiers.add(configuration.category_name, qualifiers) if qualifiers
     end
     
     # TODO Move to Index.
     #
     def generate_qualifiers_from options
-      options[:qualifiers] || options[:qualifier] && [options[:qualifier]] || [name]
+      options[:qualifiers] || options[:qualifier] && [options[:qualifier]]
     end
     
     # Loads the index from cache.
@@ -43,12 +46,6 @@ module Index
       partial.load
     end
     
-    # TODO Move to initializer?
-    #
-    def identifier
-      @identifier ||= "#{type.name} #{name}"
-    end
-    
     # Gets the weight for this token's text.
     #
     def weight token

data/lib/picky/{index/type.rb → indexed/index.rb}

@@ -1,10 +1,8 @@
-module Index
+module Indexed
   
-  # This class is for multiple types.
   #
-  # For example, you could have types books, isbn.
   #
-  class Type
+  class Index
     
     attr_reader :name, :result_type, :combinator, :categories
     
@@ -22,8 +20,10 @@ module Index
     
     # TODO Spec. Doc.
     #
-    def add_category name, options = {}
-      categories << Category.new(name, self, options)
+    def add_category category_name, options = {}
+      new_category = Category.new category_name, self, options
+      categories << new_category
+      new_category
     end
     
     #

data/lib/picky/{index/types.rb → indexed/indexes.rb}

@@ -1,11 +1,11 @@
-module Index
+module Indexed
   
-  class Types
+  class Indexes
     
-    attr_reader :types, :type_mapping
+    attr_reader :indexes, :index_mapping
     
     each_delegate :load_from_cache,
-                  :to => :types
+                  :to => :indexes
     
     def initialize
       clear
@@ -14,8 +14,8 @@ module Index
     # TODO Spec.
     #
     def clear
-      @types        = []
-      @type_mapping = {}
+      @indexes       = []
+      @index_mapping = {}
     end
     
     # TODO Spec.
@@ -26,14 +26,14 @@ module Index
     
     # TODO Spec
     #
-    def register type
-      self.types << type
-      self.type_mapping[type.name] = type
+    def register index
+      self.indexes << index
+      self.index_mapping[index.name] = index
     end
     def [] name
       name = name.to_sym
       
-      type_mapping[name]
+      index_mapping[name]
     end
     
   end

data/lib/picky/{index → indexed}/wrappers/exact_first.rb

@@ -1,13 +1,13 @@
 # encoding: utf-8
 #
-module Index
+module Indexed
   
   # FIXME and spec
   #
   module Wrappers
     
     # This index combines an exact and partial index.
-    # It serves to order the results such that exact  hits are found first.
+    # It serves to order the results such that exact hits are found first.
     #
     # TODO Need to use the right subtokens. Bake in?
     #
@@ -17,7 +17,7 @@ module Index
                :identifier,
                :name,
                :to => :@exact
-      delegate :type,
+      delegate :index,
                :category,
                :weight,
                :generate_partial_from,
@@ -32,12 +32,12 @@ module Index
         @partial = category.partial
       end
       
-      def self.wrap type_or_category
-        if type_or_category.respond_to? :categories
-          wrap_each_of type_or_category.categories
-          type_or_category
+      def self.wrap index_or_category
+        if index_or_category.respond_to? :categories
+          wrap_each_of index_or_category.categories
+          index_or_category
         else
-          new type_or_category
+          new index_or_category
         end
       end
       # TODO Do not extract categories!

data/lib/picky/indexers/no_source_specified_error.rb

@@ -1,6 +1,6 @@
 module Indexers
   
-  # Raised if no source is available on a field.
+  # Raised if no source is available on a category.
   #
   class NoSourceSpecifiedException < StandardError; end
   

data/lib/picky/indexers/serial.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+#
+module Indexers
+  
+  # The indexer defines the control flow.
+  #
+  class Serial
+    
+    attr_accessor :tokenizer, :source
+    
+    def initialize configuration, source, tokenizer
+      @configuration = configuration
+      @source        = source || raise_no_source
+      @tokenizer     = tokenizer
+    end
+    
+    # Raise a no source exception.
+    #
+    def raise_no_source
+      raise NoSourceSpecifiedException.new("No source given for #{@configuration.identifier}.")
+    end
+    
+    # Selects the original id (indexed id) and a column to process. The column data is called "token".
+    #
+    # Note: Puts together the parts first in an array, then releasing the array from time to time by joining.
+    #
+    def index
+      indexing_message
+      process
+    end
+    def process
+      comma   = ?,
+      newline = ?\n
+      
+      # TODO Move open to config?
+      #
+      # @category.prepared_index do |file|
+      #   source.harvest(@index, @category) do |indexed_id, text|
+      #     tokenizer.tokenize(text).each do |token_text|
+      #       next unless token_text
+      #       file.buffer indexed_id << comma << token_text << newline
+      #     end
+      #     file.write_maybe
+      #   end
+      # end
+      #
+      @configuration.prepared_index_file do |file|
+        result = []
+        source.harvest(@configuration.index, @configuration.category) do |indexed_id, text|
+          tokenizer.tokenize(text).each do |token_text|
+            next unless token_text
+            result << indexed_id << comma << token_text << newline
+          end
+          file.write(result.join) && result.clear if result.size > 100_000
+        end
+        file.write result.join
+      end
+    end
+    def indexing_message
+      timed_exclaim "INDEX #{@configuration.identifier}"
+    end
+    
+  end
+end

data/lib/picky/indexers/solr.rb

@@ -2,9 +2,7 @@
 #
 require 'rsolr'
 module Indexers
-  # This is an indexer in its own right.
-  #
-  # TODO Perhaps merge with the existing indexer.
+  # TODO Totally deprecated. This should be a source.
   #
   class Solr
 

data/lib/picky/indexes_api.rb

@@ -0,0 +1,41 @@
+# Comfortable API convenience class, splits methods to indexes.
+#
+class IndexesAPI
+  
+  attr_reader :indexes, :index_mapping
+  
+  delegate :reload,
+           :load_from_cache,
+           :to => :@indexed
+  
+  delegate :check_caches,
+           :find,
+           :generate_cache_only,
+           :generate_index_only,
+           :index,
+           :index_for_tests,
+           :to => :@indexing
+  
+  def initialize
+    @indexes = []
+    @index_mapping = {}
+    
+    @indexed  = Indexed::Indexes.new
+    @indexing = Indexing::Indexes.new
+  end
+  
+  def register index
+    self.indexes << index
+    self.index_mapping[index.name] = index
+    
+    @indexing.register index.indexing
+    @indexed.register  index.indexed # TODO Even necessary?
+  end
+  
+  def [] name
+    name = name.to_sym
+    
+    self.index_mapping[name]
+  end
+  
+end

data/lib/picky/indexing/bundle.rb

@@ -6,17 +6,15 @@ module Indexing
   # It does all menial tasks that have nothing to do
   # with the actual index running etc.
   #
-  # TODO Superclass?
-  #
-  class Bundle < ::Bundle
+  class Bundle < Index::Bundle
     
     attr_accessor :partial_strategy, :weights_strategy
     attr_reader   :files
     
     # Path is in which directory the cache is located.
     #
-    def initialize name, category, type, similarity_strategy, partial_strategy, weights_strategy
-      super name, category, type, similarity_strategy
+    def initialize name, configuration, similarity_strategy, partial_strategy, weights_strategy
+      super name, configuration, similarity_strategy
       
       @partial_strategy    = partial_strategy
       @weights_strategy    = weights_strategy
@@ -63,7 +61,11 @@ module Indexing
     def load_from_index_generation_message
       timed_exclaim "LOAD INDEX #{identifier}."
     end
-    # Retrieves the data into the index.
+    # Retrieves the prepared index data into the index.
+    #
+    # This is in preparation for generating
+    # derived indexes (like weights, similarity)
+    # and later dumping the optimized index.
     #
     def retrieve
       files.retrieve do |id, token|
@@ -71,6 +73,8 @@ module Indexing
         index[token] << id
       end
     end
+    # Sets up a piece of the index for the given token.
+    #
     def initialize_index_for token
       index[token] ||= []
     end
@@ -81,12 +85,14 @@ module Indexing
     #
     
     # Generates a new index (writes its index) using the
-    # given partial caching strategy.
+    # partial caching strategy of this bundle.
     #
     def generate_partial
       generator = Cacher::PartialGenerator.new self.index
       self.index = generator.generate self.partial_strategy
     end
+    # Generate a partial index from the given exact index.
+    #
     def generate_partial_from exact_index
       timed_exclaim "PARTIAL GENERATE #{identifier}."
       self.index = exact_index
@@ -108,25 +114,31 @@ module Indexing
       self.weights = generator.generate self.weights_strategy
     end
 
-    # Saves the index in a dump file.
+    # Saves the indexes in a dump file.
     #
     def dump
       dump_index
       dump_similarity
       dump_weights
     end
+    # Dumps the core index.
+    #
     def dump_index
       timed_exclaim "DUMP INDEX #{identifier}."
       files.dump_index index
     end
-    def dump_similarity
-      timed_exclaim "DUMP SIMILARITY #{identifier}."
-      files.dump_similarity similarity
-    end
+    # Dumps the weights index.
+    #
     def dump_weights
       timed_exclaim "DUMP WEIGHTS #{identifier}."
       files.dump_weights weights
     end
+    # Dumps the similarity index.
+    #
+    def dump_similarity
+      timed_exclaim "DUMP SIMILARITY #{identifier}."
+      files.dump_similarity similarity
+    end
     
     # Alerts the user if an index is missing.
     #
@@ -134,38 +146,56 @@ module Indexing
       raise_unless_index_exists
       raise_unless_similarity_exists
     end
+    # Alerts the user if one of the necessary indexes
+    # (core, weights) is missing.
+    #
     def raise_unless_index_exists
       if partial_strategy.saved?
         warn_if_index_small
         raise_unless_index_ok
       end
     end
+    # Alerts the user if the similarity
+    # index is missing (given that it's used).
+    #
     def raise_unless_similarity_exists
       if similarity_strategy.saved?
         warn_if_similarity_small
         raise_unless_similarity_ok
       end
     end
+    # Warns the user if the similarity index is small.
+    #
     def warn_if_similarity_small
       warn_cache_small :similarity if files.similarity_cache_small?
     end
+    # Alerts the user if the similarity index is not there.
+    #
     def raise_unless_similarity_ok
       raise_cache_missing :similarity unless files.similarity_cache_ok?
     end
+    
     # TODO Spec on down.
     #
+    
+    # Warns the user if the core or weights indexes are small.
+    #
     def warn_if_index_small
       warn_cache_small :index   if files.index_cache_small?
       warn_cache_small :weights if files.weights_cache_small?
     end
+    # Alerts the user if the core or weights indexes are not there.
+    #
     def raise_unless_index_ok
       raise_cache_missing :index   unless files.index_cache_ok?
       raise_cache_missing :weights unless files.weights_cache_ok?
     end
+    # Outputs a warning for the given cache.
+    #
     def warn_cache_small what
       puts "Warning: #{what} cache for #{identifier} smaller than 16 bytes."
     end
-    # Raises an appropriate error message.
+    # Raises an appropriate error message for the given cache.
     #
     def raise_cache_missing what
       raise "#{what} cache for #{identifier} missing."

data/lib/picky/indexing/category.rb

@@ -2,20 +2,17 @@ module Indexing
   
   class Category
     
-    attr_reader :name, :type, :indexed_as, :virtual, :tokenizer, :source, :exact, :partial
+    attr_reader :exact, :partial, :name, :configuration, :indexer
     
-    # TODO Dup the options?
-    #
-    def initialize name, type, options = {}
+    def initialize name, index, options = {}
       @name = name
-      @type = type
       
-      @source        = options[:source]
+      # Now we have enough info to combine the index and the category.
+      #
+      @configuration = Configuration::Index.new index, self
       
-      @tokenizer     = options[:tokenizer] || Tokenizers::Index.default
-      @indexer_class = options[:indexer]   || Indexers::Default
-      @indexed_as    = options[:as]        || name
-      @virtual       = options[:virtual]   || false # TODO What is this again?
+      @tokenizer = options[:tokenizer] || Tokenizers::Index.default
+      @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer #, :as => options[:as] # TODO option as.
       
       # TODO Push into Bundle.
       #
@@ -23,28 +20,12 @@ module Indexing
       weights    = options[:weights]    || Cacher::Weights::Default
       similarity = options[:similarity] || Cacher::Similarity::Default
       
-      @exact   = options[:exact_indexing_bundle]   || Bundle.new(:exact,   self, type, similarity, Cacher::Partial::None.new, weights)
-      @partial = options[:partial_indexing_bundle] || Bundle.new(:partial, self, type, Cacher::Similarity::None.new, partial, weights)
-      
-      # TODO Move to Query.
-      #
-      # @remove          = options[:remove]        || false
-      # @filter          = options[:filter]        || true
-      
-      @options = options # TODO Remove?
+      @exact   = options[:exact_indexing_bundle]   || Bundle.new(:exact,   configuration, similarity, Cacher::Partial::None.new, weights)
+      @partial = options[:partial_indexing_bundle] || Bundle.new(:partial, configuration, Cacher::Similarity::None.new, partial, weights)
     end
     
-    # TODO Move to initializer?
-    #
-    def identifier
-      @identifier ||= "#{type.name} #{name}"
-    end
-    
-    # Note: Most of the time the source of the type is used.
-    #
-    def source
-      @source || type.source
-    end
+    delegate :identifier, :prepare_index_directory, :to => :configuration
+    delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
     
     # TODO Spec.
     #
@@ -68,16 +49,16 @@ module Indexing
       exact.delete
       partial.delete
     end
-    # def create_directory_structure
-    #   timed_exclaim "Creating directory structure for #{identifier}."
-    #   exact.create_directory
-    #   partial.create_directory
-    # end
+    
+    def index
+      prepare_index_directory
+      indexer.index
+    end
     
     # Generates all caches for this category.
     #
     def cache
-      prepare_cache_directory
+      prepare_index_directory
       generate_caches
     end
     def generate_caches
@@ -101,34 +82,6 @@ module Indexing
       partial.dump
     end
     
-    # TODO Partially move to type. Duplicate Code in indexers/field.rb.
-    #
-    # TODO Use the Files object.
-    #
-    def search_index_root
-      File.join PICKY_ROOT, 'index'
-    end
-    def cache_directory
-      File.join search_index_root, PICKY_ENVIRONMENT, type.name.to_s
-    end
-    def search_index_file_name
-      File.join cache_directory, "prepared_#{name}_index.txt"
-    end
-    def index
-      prepare_cache_directory
-      # files.create_directory # TODO Make this possible!
-      indexer.index
-    end
-    def prepare_cache_directory
-      FileUtils.mkdir_p cache_directory
-    end
-    def indexer
-      @indexer || @indexer = @indexer_class.new(type, self)
-    end
-    def virtual?
-      !!virtual
-    end
-    
   end
   
 end

data/lib/picky/indexing/{type.rb → index.rb}

@@ -1,6 +1,6 @@
 module Indexing
   
-  class Type
+  class Index
     
     attr_reader :name, :source, :categories, :after_indexing
     
@@ -30,8 +30,18 @@ module Indexing
     
     # TODO Spec. Doc.
     #
-    def add_category name, options = {}
-      categories << Category.new(name, self, options)
+    def add_category category_name, options = {}
+      options = default_category_options.merge options
+      
+      new_category = Category.new category_name, self, options
+      categories << new_category
+      new_category
+    end
+    
+    # By default, the category uses the index's source.
+    #
+    def default_category_options
+      { :source => @source }
     end
     
     # Indexing.