picky 0.10.5 → 0.11.0

data/lib/picky/alias_instances.rb

@@ -0,0 +1 @@
+Indexes = Types.new

data/lib/picky/application.rb

@@ -21,13 +21,6 @@ class Application
     # Routes.
     #
     delegate :route, :root, :to => :routing
-    # Index, Field.
-    #
-    # TODO Rename category.
-    #
-    delegate :field, :to => :indexing
-    def category *args; indexing.field *args;        end
-    def index *args;    indexing.define_index *args; end
     
     # An application simply delegates to the routing to handle a request.
     #
@@ -44,6 +37,12 @@ class Application
       @queries ||= Configuration::Queries.new
     end
     
+    # "API".
+    #
+    def index *args
+      ::Type.new *args
+    end
+    
     # Finalize the subclass as soon as it
     # has finished loading.
     #

data/lib/picky/bundle.rb

@@ -0,0 +1,31 @@
+class Bundle
+  
+  attr_reader   :identifier, :files
+  attr_accessor :index, :weights, :similarity, :similarity_strategy
+  
+  delegate :[], :[]=, :clear, :to => :index
+  
+  def initialize name, category, type, similarity_strategy
+    @identifier = "#{name}: #{type.name} #{category.name}"
+    
+    @index      = {}
+    @weights    = {}
+    @similarity = {}
+    
+    @similarity_strategy = similarity_strategy
+    
+    # TODO inject files.
+    #
+    # TODO Move Files somewhere. Shared?
+    #
+    @files = Index::Files.new name, category.name, type.name
+  end
+  
+  # Get a list of similar texts.
+  #
+  def similar text
+    code = similarity_strategy.encoded text
+    code && @similarity[code] || []
+  end
+  
+end

data/lib/picky/configuration/indexes.rb

@@ -1,3 +1,5 @@
+# TODO Remove?
+#
 module Configuration
   
   # Describes the container for all index configurations.
@@ -14,48 +16,35 @@ module Configuration
       Tokenizers::Index.default = Tokenizers::Index.new(options)
     end
     
-    # TODO Rewrite all this configuration handling.
+    # TODO Move this to … where?
     #
-    def define_index name, source, *fields
-      new_type = Type.new name, source, *fields
-      types << new_type
-      ::Indexes.configuration ||= self
-      
-      generated = new_type.generate
-      ::Indexes.add generated
-      generated
-    end
-    def field name, options = {}
-      Field.new name, options
-    end
-    
-    #
-    #
-    def take_snapshot *type_names
-      only_if_included_in type_names do |type|
-        type.take_snapshot
-      end
-    end
-    def index *type_names
-      only_if_included_in type_names do |type|
-        type.index
-      end
-    end
-    def index_solr *type_names
-      only_if_included_in type_names do |type|
-        type.index_solr
-      end
-    end
-    
-    #
-    #
-    def only_if_included_in type_names = []
-      type_names = types.map(&:name) if type_names.empty?
-      types.each do |type|
-        next unless type_names.include?(type.name)
-        yield type
-      end
-    end
+    # #
+    # #
+    # def take_snapshot *type_names
+    #   only_if_included_in type_names do |type|
+    #     type.take_snapshot
+    #   end
+    # end
+    # def index *type_names
+    #   only_if_included_in type_names do |type|
+    #     type.index
+    #   end
+    # end
+    # def index_solr *type_names
+    #   only_if_included_in type_names do |type|
+    #     type.index_solr
+    #   end
+    # end
+    # 
+    # #
+    # #
+    # def only_if_included_in type_names = []
+    #   type_names = types.map(&:name) if type_names.empty?
+    #   types.each do |type|
+    #     next unless type_names.include?(type.name)
+    #     yield type
+    #   end
+    # end
     
   end
 

data/lib/picky/configuration/type.rb

@@ -2,53 +2,19 @@ module Configuration
   class Type
     attr_reader :name,
                 :source,
-                :fields,
-                :after_indexing,
                 :result_type,
-                :ignore_unassigned_tokens,
-                :solr
-    def initialize name, source, *fields, options
-      if Configuration::Field === options
-        fields << options
-        options = {}
-      end
-      
+                :after_indexing,
+                :ignore_unassigned_tokens
+    def initialize name, source, options
       @name                     = name
       @source                   = source
-                                  # dup, if field is reused. TODO Rewrite.
-      @fields                   = fields.map { |field| field = field.dup; field.type = self; field }
       
-      @after_indexing           = options[:after_indexing]
       @result_type              = options[:result_type] || name
-      @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false       # TODO Move to query?
-      # @solr                     = options[:solr] || nil
+      @after_indexing           = options[:after_indexing] # Where do I use this?
+      @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false # TODO Move to query?
     end
     def generate
-      categories = fields.map { |field| field.generate }
-      Index::Type.new name, result_type, ignore_unassigned_tokens, *categories
-    end
-    def take_snapshot
-      source.take_snapshot self
-    end
-    def index
-      fields.each do |field|
-        field.index
-      end
-    end
-    # def solr_fields
-    #   solr ? fields.select { |field| !field.virtual? } : []
-    # end
-    # # TODO Delegate to Solr handler.
-    # #
-    # def index_solr
-    #   return unless solr
-    #   @indexer = Indexers::Solr.new self
-    #   @indexer.index
-    # end
-    # TODO Spec!
-    #
-    def connect_backend
-      @source.connect_backend
+      Index::Type.new name, source, result_type, ignore_unassigned_tokens
     end
   end
 end

data/lib/picky/ext/maybe_compile.rb

@@ -0,0 +1,9 @@
+begin
+  require File.expand_path '../ruby19/performant', __FILE__
+rescue LoadError
+  require File.expand_path '../ruby19/extconf.rb', __FILE__
+  Dir.chdir File.expand_path('../ruby19', __FILE__) do
+    %x{ ruby extconf.rb && make }
+  end
+  retry
+end

data/lib/picky/index/bundle.rb

@@ -8,35 +8,7 @@ module Index
   #
   # Delegates file handling and checking to a Index::Files object.
   #
-  class Bundle
-    
-    attr_reader   :identifier, :category
-    attr_accessor :index,            :weights,          :similarity
-    attr_accessor :partial_strategy, :weights_strategy, :similarity_strategy
-    attr_reader   :files
-    
-    delegate :[], :[]=, :clear, :to => :index
-    delegate :raise_unless_cache_exists, :to => :checker
-    
-    # Path is in which directory the cache is located.
-    #
-    def initialize name, category, type, partial_strategy, weights_strategy, similarity_strategy
-      @identifier = "#{name}: #{type.name} #{category.name}"
-      
-      @index      = {}
-      @weights    = {}
-      @similarity = {}
-      
-      # TODO Used in weights, try to remove!
-      #
-      @category = category
-      
-      @partial_strategy    = partial_strategy
-      @weights_strategy    = weights_strategy
-      @similarity_strategy = similarity_strategy
-      
-      @files   = Files.new name, category.name, type.name
-    end
+  class Bundle < ::Bundle
     
     # Get the ids for the text.
     #
@@ -48,43 +20,6 @@ module Index
     def weight text
       @weights[text]
     end
-    # Get a list of similar texts.
-    #
-    def similar text
-      code = similarity_strategy.encoded text
-      code && @similarity[code] || []
-    end
-    
-    # Generation
-    #
-    
-    # This method
-    # * loads the base index from the db
-    # * generates derived indexes
-    # * dumps all the indexes into files
-    #
-    def generate_caches_from_source
-      load_from_index_file
-      generate_caches_from_memory
-    end
-    # Generates derived indexes from the index and dumps.
-    #
-    # Note: assumes that there is something in the index
-    #
-    def generate_caches_from_memory
-      cache_from_memory_generation_message
-      generate_derived
-    end
-    def cache_from_memory_generation_message
-      timed_exclaim "CACHE FROM MEMORY #{identifier}."
-    end
-    
-    # Generates the weights and similarity from the main index.
-    #
-    def generate_derived
-      generate_weights
-      generate_similarity
-    end
     
     # Load the data from the db.
     #
@@ -108,59 +43,6 @@ module Index
       index[token] ||= []
     end
     
-    # Generators.
-    #
-    # TODO Move somewhere more fitting.
-    #
-    
-    # Generates a new index (writes its index) using the
-    # given partial caching strategy.
-    #
-    def generate_partial
-      generator = Cacher::PartialGenerator.new self.index
-      self.index = generator.generate self.partial_strategy
-    end
-    def generate_partial_from exact_index
-      timed_exclaim "PARTIAL GENERATE #{identifier}."
-      self.index = exact_index
-      self.generate_partial
-      self
-    end
-    # Generates a new similarity index (writes its index) using the
-    # given similarity caching strategy.
-    #
-    def generate_similarity
-      generator = Cacher::SimilarityGenerator.new self.index
-      self.similarity = generator.generate self.similarity_strategy
-    end
-    # Generates a new weights index (writes its index) using the
-    # given weight caching strategy.
-    #
-    def generate_weights
-      generator = Cacher::WeightsGenerator.new self.index
-      self.weights = generator.generate self.weights_strategy
-    end
-
-    # Saves the index in a dump file.
-    #
-    def dump
-      dump_index
-      dump_similarity
-      dump_weights
-    end
-    def dump_index
-      timed_exclaim "DUMP INDEX #{identifier}."
-      files.dump_index index
-    end
-    def dump_similarity
-      timed_exclaim "DUMP SIMILARITY #{identifier}."
-      files.dump_similarity similarity
-    end
-    def dump_weights
-      timed_exclaim "DUMP WEIGHTS #{identifier}."
-      files.dump_weights weights
-    end
-    
     # Loads all indexes into this category.
     #
     def load
@@ -178,25 +60,5 @@ module Index
       self.weights = files.load_weights
     end
     
-    # Alerts the user if an index is missing.
-    #
-    def raise_unless_cache_exists
-      warn_cache_small :index      if files.index_cache_small?
-      warn_cache_small :similarity if files.similarity_cache_small?
-      warn_cache_small :weights    if files.weights_cache_small?
-
-      raise_cache_missing :index      unless files.index_cache_ok?
-      raise_cache_missing :similarity unless files.similarity_cache_ok?
-      raise_cache_missing :weights    unless files.weights_cache_ok?
-    end
-    def warn_cache_small what
-      puts "#{what} cache for #{identifier} smaller than 16 bytes."
-    end
-    # Raises an appropriate error message.
-    #
-    def raise_cache_missing what
-      raise "#{what} cache for #{identifier} missing."
-    end
-    
   end
 end

data/lib/picky/{query/combinator.rb → index/categories.rb}

@@ -1,28 +1,26 @@
-module Query
+module Index
   
-  # Combines tokens and category indexes into combinations.
-  #
-  class Combinator
+  class Categories
     
-    attr_reader :categories, :category_hash
-    attr_reader :ignore_unassigned_tokens # TODO Should this actually be determined by the query? Probably, yes.
+    attr_reader :categories, :category_hash, :ignore_unassigned_tokens
     
-    def initialize categories, options = {}
-      @categories               = categories
-      @category_hash            = hashify categories
+    each_delegate :load_from_cache,
+                  :to => :categories
+    
+    def initialize options = {}
+      clear
       
       @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
     end
     
-    # TODO Move somewhere else.
-    #
-    # TODO Or use active_support's?
-    #
-    def hashify category_array
-      category_array.inject({}) do |hash, category|
-        hash[category.name] = [category]
-        hash
-      end
+    def clear
+      @categories    = []
+      @category_hash = {}
+    end
+    
+    def << category
+      categories << category
+      category_hash[category.name] = [category] # TODO Why an Array?
     end
     
     #

data/lib/picky/index/category.rb

@@ -15,19 +15,30 @@ module Index
       @name = name
       @type = type
       
-      partial    = options[:partial]    || Cacher::Partial::Default
-      weights    = options[:weights]    || Cacher::Weights::Default
       similarity = options[:similarity] || Cacher::Similarity::Default
       
-      @exact   = options[:exact_bundle]   || Bundle.new(:exact,   self, type, Cacher::Partial::None.new, weights, similarity)
-      @partial = options[:partial_bundle] || Bundle.new(:partial, self, type, partial, weights, Cacher::Similarity::None.new)
+      @exact   = options[:exact_index_bundle]   || Bundle.new(:exact,   self, type, similarity)
+      @partial = options[:partial_index_bundle] || Bundle.new(:partial, self, type, similarity)
       
       @exact   = exact_lambda.call(@exact, @partial)   if exact_lambda   = options[:exact_lambda]
       @partial = partial_lambda.call(@exact, @partial) if partial_lambda = options[:partial_lambda]
+      
+      # Extract?
+      #
+      qualifiers = generate_qualifiers_from options
+      Query::Qualifiers.add(name, qualifiers) if qualifiers
+    end
+    
+    # TODO Move to Index.
+    #
+    def generate_qualifiers_from options
+      options[:qualifiers] || options[:qualifier] && [options[:qualifier]] || [name]
     end
     
     # Loads the index from cache.
     #
+    # TODO Metaprogram delegation? each_delegate?
+    #
     def load_from_cache
       timed_exclaim "Loading index #{identifier}."
       exact.load
@@ -40,67 +51,30 @@ module Index
       @identifier ||= "#{type.name} #{name}"
     end
     
-    # Generates all caches for this category.
-    #
-    def generate_caches
-      generate_caches_from_source
-      generate_partial
-      generate_caches_from_memory
-      dump_caches
-      timed_exclaim "CACHE FINISHED #{identifier}."
-    end
-    def generate_caches_from_source
-      exact.generate_caches_from_source
-    end
-    def generate_partial
-      partial.generate_partial_from exact.index
-    end
-    def generate_caches_from_memory
-      partial.generate_caches_from_memory
-    end
-    def dump_caches
-      exact.dump
-      partial.dump
-    end
-    
-    # Used for testing.
-    #
-    def generate_indexes_from_exact_index
-      generate_derived_exact
-      generate_partial
-      generate_derived_partial
-    end
-    def generate_derived_exact
-      exact.generate_derived
-    end
-    def generate_derived_partial
-      partial.generate_derived
-    end
-
     # Gets the weight for this token's text.
     #
     def weight token
       bundle_for(token).weight token.text
     end
-
+    
     # Gets the ids for this token's text.
     #
     def ids token
       bundle_for(token).ids token.text
     end
-
+    
     # Returns the right index bundle for this token.
     #
     def bundle_for token
       token.partial? ? partial : exact
     end
-
+    
     #
     #
     def combination_for token
       weight(token) && ::Query::Combination.new(token, self)
     end
-
+    
   end
-
+  
 end

data/lib/picky/index/type.rb

@@ -6,26 +6,30 @@ module Index
   #
   class Type
     
-    attr_reader :name, :result_type, :categories, :combinator
+    attr_reader :name, :result_type, :combinator, :categories
     
-    each_delegate :generate_caches, :load_from_cache, :to => :categories
+    delegate :load_from_cache,
+             :to => :categories
     
-    # TODO Use config
-    #
-    def initialize name, result_type, ignore_unassigned_tokens, *categories
-      @name        = name
-      @result_type = result_type # TODO Move.
-      @categories  = categories # for each_delegate
-      @combinator  = combinator_for categories, ignore_unassigned_tokens
+    def initialize name, options = {}
+      @name                     = name
+      
+      @result_type              = options[:result_type] || name
+      ignore_unassigned_tokens  = options[:ignore_unassigned_tokens] || false # TODO Move to query, somehow.
+      
+      @categories = Categories.new ignore_unassigned_tokens: ignore_unassigned_tokens
     end
-    def combinator_for categories, ignore_unassigned_tokens
-       Query::Combinator.new @categories, ignore_unassigned_tokens: ignore_unassigned_tokens
+    
+    # TODO Spec. Doc.
+    #
+    def add_category name, options = {}
+      categories << Category.new(name, self, options)
     end
     
     #
     #
     def possible_combinations token
-      @combinator.possible_combinations_for token
+      categories.possible_combinations_for token
     end
     
   end

data/lib/picky/index/types.rb

@@ -0,0 +1,41 @@
+module Index
+  
+  class Types
+    
+    attr_reader :types, :type_mapping
+    
+    each_delegate :load_from_cache,
+                  :to => :types
+    
+    def initialize
+      clear
+    end
+    
+    # TODO Spec.
+    #
+    def clear
+      @types        = []
+      @type_mapping = {}
+    end
+    
+    # TODO Spec.
+    #
+    def reload
+      load_from_cache
+    end
+    
+    # TODO Spec
+    #
+    def register type
+      self.types << type
+      self.type_mapping[type.name] = type
+    end
+    def [] name
+      name = name.to_sym
+      
+      type_mapping[name]
+    end
+    
+  end
+  
+end

data/lib/picky/index/wrappers/exact_first.rb

@@ -2,6 +2,8 @@
 #
 module Index
   
+  # FIXME and spec
+  #
   module Wrappers
     
     # This index combines an exact and partial index.
@@ -38,8 +40,10 @@ module Index
           new type_or_category
         end
       end
+      # TODO Do not extract categories!
+      #
       def self.wrap_each_of categories
-        categories.collect! { |category| new(category) }
+        categories.categories.collect! { |category| new(category) }
       end
       
       def ids text