picky 2.5.2 → 2.6.0

data/lib/picky/indexing/bundle/super_base.rb

@@ -0,0 +1,61 @@
+# TODO Merge into Base, extract common with Indexed::Base.
+#
+module Indexing # :nodoc:all
+  # A Bundle is a number of indexes
+  # per [index, category] combination.
+  #
+  # At most, there are three indexes:
+  # * *core* index (always used)
+  # * *weights* index (always used)
+  # * *similarity* index (used with similarity)
+  #
+  # In Picky, indexing is separated from the index
+  # handling itself through a parallel structure.
+  #
+  # Both use methods provided by this base class, but
+  # have very different goals:
+  #
+  # * *Indexing*::*Bundle* is just concerned with creating index files
+  #   and providing helper functions to e.g. check the indexes.
+  #
+  # * *Index*::*Bundle* is concerned with loading these index files into
+  #   memory and looking up search data as fast as possible.
+  #
+  module Bundle
+
+    class SuperBase
+
+      attr_reader   :identifier, :files
+      attr_accessor :index, :weights, :similarity, :configuration, :similarity_strategy
+
+      delegate :clear,    :to => :index
+      delegate :[], :[]=, :to => :configuration
+
+      def initialize name, category, similarity_strategy
+        @identifier    = "#{category.identifier}:#{name}"
+        @files         = Backend::Files.new name, category
+
+        @index         = {}
+        @weights       = {}
+        @similarity    = {}
+        @configuration = {} # A hash with config options.
+
+        @similarity_strategy = similarity_strategy
+      end
+
+      # Get a list of similar texts.
+      #
+      # Note: Does not return itself.
+      #
+      def similar text
+        code = similarity_strategy.encoded text
+        similar_codes = code && @similarity[code]
+        similar_codes.delete text if similar_codes
+        similar_codes || []
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/indexing/wrappers/category/location.rb

@@ -0,0 +1,25 @@
+module Indexing
+  module Wrappers
+    module Category
+
+      module Location
+
+        def self.install_on category, grid, precision = 1
+          new_source = Sources::Wrappers::Location.new category.source, grid, precision
+
+          category.class_eval do
+            def tokenizer
+              @tokenizer ||= Tokenizers::Index.new
+            end
+            define_method :source do
+              new_source
+            end
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/picky/interfaces/live_parameters.rb

@@ -141,29 +141,29 @@ module Interfaces
     # TODO Move to Interface object.
     #
     def querying_removes_characters
-      regexp = Internals::Tokenizers::Query.default.instance_variable_get :@removes_characters_regexp
+      regexp = Tokenizers::Query.default.instance_variable_get :@removes_characters_regexp
       regexp && regexp.source
     end
     def querying_removes_characters= new_value
-      Internals::Tokenizers::Query.default.instance_variable_set(:@removes_characters_regexp, %r{#{new_value}})
+      Tokenizers::Query.default.instance_variable_set(:@removes_characters_regexp, %r{#{new_value}})
     end
     def querying_stopwords
-      regexp = Internals::Tokenizers::Query.default.instance_variable_get :@remove_stopwords_regexp
+      regexp = Tokenizers::Query.default.instance_variable_get :@remove_stopwords_regexp
       regexp && regexp.source
     end
     def querying_stopwords= new_value
-      Internals::Tokenizers::Query.default.instance_variable_set(:@remove_stopwords_regexp, %r{#{new_value}})
+      Tokenizers::Query.default.instance_variable_set(:@remove_stopwords_regexp, %r{#{new_value}})
     end
     def querying_splits_text_on
-      splits = Internals::Tokenizers::Query.default.instance_variable_get :@splits_text_on
+      splits = Tokenizers::Query.default.instance_variable_get :@splits_text_on
       splits && splits.respond_to?(:source) ? splits.source : splits
     end
     def querying_splits_text_on= new_value
-      splits = Internals::Tokenizers::Query.default.instance_variable_get :@splits_text_on
+      splits = Tokenizers::Query.default.instance_variable_get :@splits_text_on
       if splits.respond_to?(:source)
-        Internals::Tokenizers::Query.default.instance_variable_set(:@splits_text_on, %r{#{new_value}})
+        Tokenizers::Query.default.instance_variable_set(:@splits_text_on, %r{#{new_value}})
       else
-        Internals::Tokenizers::Query.default.instance_variable_set(:@splits_text_on, new_value)
+        Tokenizers::Query.default.instance_variable_set(:@splits_text_on, new_value)
       end
     end
 

data/lib/picky/loader.rb

@@ -25,9 +25,6 @@ module Loader # :nodoc:all
   def self.load_relative filename_without_rb
     load File.join(File.dirname(__FILE__), "#{filename_without_rb}.rb")
   end
-  def self.load_internals filename_without_rb
-    load File.join(File.dirname(__FILE__), "internals/#{filename_without_rb}.rb")
-  end
 
   def self.load_user filename
     load File.join(PICKY_ROOT, "#{filename}.rb")
@@ -76,7 +73,7 @@ module Loader # :nodoc:all
 
     # TODO Rewrite
     #
-    Internals::Query::Qualifiers.instance.prepare
+    Query::Qualifiers.instance.prepare
 
     exclaim "Application #{Application.apps.map(&:name).join(', ')} loaded."
   end
@@ -85,143 +82,132 @@ module Loader # :nodoc:all
   # (Not for the user)
   #
   def self.load_framework_internals
-    load_relative 'internals'
-
     # Load compiled C code.
     #
-    load_internals 'ext/maybe_compile'
+    load_relative 'ext/maybe_compile'
 
     # Load extensions.
     #
-    load_internals 'extensions/object'
-    load_internals 'extensions/array'
-    load_internals 'extensions/symbol'
-    load_internals 'extensions/module'
-    load_internals 'extensions/hash'
+    load_relative 'extensions/object'
+    load_relative 'extensions/array'
+    load_relative 'extensions/symbol'
+    load_relative 'extensions/module'
+    load_relative 'extensions/class'
+    load_relative 'extensions/hash'
 
     # Requiring Helpers
     #
-    load_internals 'helpers/measuring'
+    load_relative 'helpers/measuring'
 
     # Calculations.
     #
-    load_internals 'calculations/location'
+    load_relative 'calculations/location'
 
     # Index generation strategies.
     #
-    load_internals 'indexers/base'
-    load_internals 'indexers/serial'
-    load_internals 'indexers/parallel'
+    load_relative 'indexers/base'
+    load_relative 'indexers/serial'
+    load_relative 'indexers/parallel'
 
     # Generators.
     #
-    load_internals 'generators/strategy'
+    load_relative 'generators/strategy'
 
     # Partial index generation strategies.
     #
-    load_internals 'generators/partial/strategy'
-    load_internals 'generators/partial/none'
-    load_internals 'generators/partial/substring'
-    load_internals 'generators/partial/default'
+    load_relative 'generators/partial/strategy'
+    load_relative 'generators/partial/none'
+    load_relative 'generators/partial/substring'
+    load_relative 'generators/partial/default'
 
     # Weight index generation strategies.
     #
-    load_internals 'generators/weights/strategy'
-    load_internals 'generators/weights/logarithmic'
-    load_internals 'generators/weights/default'
+    load_relative 'generators/weights/strategy'
+    load_relative 'generators/weights/logarithmic'
+    load_relative 'generators/weights/default'
 
     # Similarity index generation strategies.
     #
-    load_internals 'generators/similarity/strategy'
-    load_internals 'generators/similarity/none'
-    load_internals 'generators/similarity/phonetic'
-    load_internals 'generators/similarity/metaphone'
-    load_internals 'generators/similarity/double_metaphone'
-    load_internals 'generators/similarity/soundex'
-    load_internals 'generators/similarity/default'
+    load_relative 'generators/similarity/strategy'
+    load_relative 'generators/similarity/none'
+    load_relative 'generators/similarity/phonetic'
+    load_relative 'generators/similarity/metaphone'
+    load_relative 'generators/similarity/double_metaphone'
+    load_relative 'generators/similarity/soundex'
+    load_relative 'generators/similarity/default'
 
     # Index generators.
     #
-    load_internals 'generators/base'
-    load_internals 'generators/partial_generator'
-    load_internals 'generators/weights_generator'
-    load_internals 'generators/similarity_generator'
-
-    # Shared index elements.
-    #
-    load_internals 'shared/category'
+    load_relative 'generators/base'
+    load_relative 'generators/partial_generator'
+    load_relative 'generators/weights_generator'
+    load_relative 'generators/similarity_generator'
 
     # Index store handling.
     #
-    load_internals 'index/backend'
+    load_relative 'backend/backend'
 
-    load_internals 'index/redis'
-    load_internals 'index/redis/basic'
-    load_internals 'index/redis/list_hash'
-    load_internals 'index/redis/string_hash'
+    load_relative 'backend/redis'
+    load_relative 'backend/redis/basic'
+    load_relative 'backend/redis/list_hash'
+    load_relative 'backend/redis/string_hash'
 
-    load_internals 'index/file/basic'
-    load_internals 'index/file/text'
-    load_internals 'index/file/marshal'
-    load_internals 'index/file/json'
-
-    load_internals 'index/files'
+    load_relative 'backend/file/basic'
+    load_relative 'backend/file/text'
+    load_relative 'backend/file/marshal'
+    load_relative 'backend/file/json'
 
+    load_relative 'backend/files'
+    
     # Indexing and Indexed things.
     #
-    load_internals 'indexing/bundle/super_base' # TODO Remove.
-    load_internals 'indexing/bundle/base'
-    load_internals 'indexing/bundle/memory'
-    load_internals 'indexing/bundle/redis'
-    load_internals 'indexing/category'
-    # load_internals 'indexing/categories'
-    load_internals 'indexing/index'
+    load_relative 'indexing/bundle/super_base' # TODO Remove.
+    load_relative 'indexing/bundle/base'
+    load_relative 'indexing/bundle/memory'
+    load_relative 'indexing/bundle/redis'
 
-    load_internals 'indexing/wrappers/category/location'
+    load_relative 'indexing/wrappers/category/location'
 
-    load_internals 'indexed/bundle/base'
-    load_internals 'indexed/bundle/memory'
-    load_internals 'indexed/bundle/redis'
-    load_internals 'indexed/category'
-    load_internals 'indexed/categories'
-    load_internals 'indexed/index'
+    load_relative 'indexed/bundle/base'
+    load_relative 'indexed/bundle/memory'
+    load_relative 'indexed/bundle/redis'
 
-    load_internals 'indexed/wrappers/exact_first'
+    load_relative 'indexed/wrappers/exact_first'
 
     # Bundle Wrapper
     #
-    load_internals 'indexed/wrappers/bundle/wrapper'
-    load_internals 'indexed/wrappers/bundle/calculation'
-    load_internals 'indexed/wrappers/bundle/location'
+    load_relative 'indexed/wrappers/bundle/wrapper'
+    load_relative 'indexed/wrappers/bundle/calculation'
+    load_relative 'indexed/wrappers/bundle/location'
 
-    load_internals 'indexed/wrappers/category/location'
+    load_relative 'indexed/wrappers/category/location'
 
     # Tokens.
     #
-    load_internals 'query/token'
-    load_internals 'query/tokens'
+    load_relative 'query/token'
+    load_relative 'query/tokens'
 
     # Tokenizers types.
     #
-    load_internals 'tokenizers/base'
-    load_internals 'tokenizers/index'
-    load_internals 'tokenizers/query'
+    load_relative 'tokenizers/base'
+    load_relative 'tokenizers/index'
+    load_relative 'tokenizers/query'
 
     # Query combinations, qualifiers, weigher.
     #
-    load_internals 'query/combination'
-    load_internals 'query/combinations/base'
-    load_internals 'query/combinations/memory'
-    load_internals 'query/combinations/redis'
+    load_relative 'query/combination'
+    load_relative 'query/combinations/base'
+    load_relative 'query/combinations/memory'
+    load_relative 'query/combinations/redis'
 
-    load_internals 'query/allocation'
-    load_internals 'query/allocations'
+    load_relative 'query/allocation'
+    load_relative 'query/allocations'
 
-    load_internals 'query/qualifiers'
+    load_relative 'query/qualifiers'
 
-    load_internals 'query/weights'
+    load_relative 'query/weights'
 
-    load_internals 'query/indexes'
+    load_relative 'query/indexes'
 
     # Configuration.
     #
@@ -229,14 +215,14 @@ module Loader # :nodoc:all
 
     # Adapters.
     #
-    load_internals 'adapters/rack/base'
-    load_internals 'adapters/rack/query'
-    load_internals 'adapters/rack/live_parameters'
-    load_internals 'adapters/rack'
+    load_relative 'adapters/rack/base'
+    load_relative 'adapters/rack/query'
+    load_relative 'adapters/rack/live_parameters'
+    load_relative 'adapters/rack'
 
     # Routing.
     #
-    load_internals 'frontend_adapters/rack'
+    load_relative 'frontend_adapters/rack'
   end
   # Loads the user interface parts.
   #
@@ -271,15 +257,23 @@ module Loader # :nodoc:all
 
     # API.
     #
+    load_relative 'category'
+    load_relative 'category_indexed'
+    load_relative 'category_indexing'
+    
+    load_relative 'categories'
+    load_relative 'categories_indexed'
+    load_relative 'categories_indexing'
+    
     load_relative 'index/base'
+    load_relative 'index/base_indexed'
+    load_relative 'index/base_indexing'
     load_relative 'index/memory'
     load_relative 'index/redis'
-
-    load_relative 'indexing/indexes'
-    load_relative 'indexed/indexes'
-
-    load_relative 'index_bundle'
-    load_relative 'aliases'
+    
+    load_relative 'indexes'
+    load_relative 'indexes_indexed'
+    load_relative 'indexes_indexing'
 
     # Results.
     #

data/lib/picky/query/allocation.rb

@@ -0,0 +1,84 @@
+module Query
+
+  # An allocation has a number of combinations:
+  # [token, index] [other_token, other_index], ...
+  #
+  class Allocation # :nodoc:all
+
+    attr_reader :count, :ids, :score, :combinations, :result_identifier
+  
+    #
+    #
+    def initialize combinations, result_identifier
+      @combinations      = combinations
+      @result_identifier = result_identifier
+    end
+
+    def hash
+      @combinations.hash
+    end
+    def eql? other_allocation
+      true # FIXME
+      # @combinations.eql? other_allocation.combinations
+    end
+
+    # Scores its combinations and caches the result.
+    #
+    def calculate_score weights
+      @score ||= @combinations.calculate_score(weights)
+    end
+
+    # Asks the combinations for the (intersected) ids.
+    #
+    def calculate_ids amount, offset
+      @combinations.ids amount, offset # Calculate as many ids as are necessary.
+    end
+
+    # This starts the searching process.
+    #
+    def process! amount, offset
+      ids    = calculate_ids amount, offset
+      @count = ids.size                         # cache the count before throwing away the ids
+      @ids   = ids.slice!(offset, amount) || [] # slice out the relevant part
+    end
+
+    #
+    #
+    def keep identifiers = [] # categories
+      @combinations.keep identifiers
+    end
+    #
+    #
+    def remove identifiers = [] # categories
+      @combinations.remove identifiers
+    end
+
+    # Sort highest score first.
+    #
+    def <=> other_allocation
+       other_allocation.score <=> self.score
+    end
+
+    # Transform the allocation into result form.
+    #
+    def to_result
+      [self.result_identifier, self.score, self.count, @combinations.to_result, self.ids] if self.count > 0
+    end
+
+    # Json representation of this allocation.
+    #
+    # Note: Delegates to to_result.
+    #
+    def to_json
+      to_result.to_json
+    end
+
+    #
+    #
+    def to_s
+      "Allocation: #{to_result.join(', ')}"
+    end
+
+  end
+  
+end

data/lib/picky/query/allocations.rb

@@ -0,0 +1,114 @@
+module Query
+  # Container class for allocations.
+  #
+  class Allocations # :nodoc:all
+
+    delegate :each, :inject, :empty?, :size, :to => :@allocations
+    attr_reader :total
+
+    def initialize allocations = []
+      @allocations = allocations
+    end
+
+    # Score each allocation.
+    #
+    def calculate_score weights
+      @allocations.each do |allocation|
+        allocation.calculate_score weights
+      end
+    end
+    # Sort the allocations.
+    #
+    def sort!
+      @allocations.sort!
+    end
+
+    # Reduces the amount of allocations to x.
+    #
+    def reduce_to amount
+      @allocations = @allocations.shift amount
+    end
+
+    # Keeps combinations.
+    #
+    # Only those passed in remain.
+    #
+    def keep identifiers = []
+      @allocations.each { |allocation| allocation.keep identifiers } unless identifiers.empty?
+    end
+    # Removes combinations.
+    #
+    # Only those passed in are removed.
+    #
+    def remove identifiers = []
+      @allocations.each { |allocation| allocation.remove identifiers } unless identifiers.empty?
+    end
+
+    # Returns the top amount ids.
+    #
+    def ids amount = 20
+      @allocations.inject([]) do |total, allocation|
+        total.size >= amount ? (return total.shift(amount)) : total + allocation.ids
+      end
+    end
+
+    # This is the main method of this class that will replace ids and count.
+    #
+    # What it does is calculate the ids and counts of its allocations
+    # for being used in the results. It also calculates the total
+    #
+    # Parameters:
+    #  * amount: the amount of ids to calculate
+    #  * offset: the offset from where in the result set to take the ids
+    #
+    # Note: With an amount of 0, an offset > 0 doesn't make much
+    #       sense, as seen in the live search.
+    #
+    # Note: Each allocation caches its count, but not its ids (thrown away).
+    #       The ids are cached in this class.
+    #
+    # Note: It's possible that no ids are returned by an allocation, but a count. (In case of an offset)
+    #
+    def process! amount, offset = 0
+      @total = 0
+      current_offset = 0
+      @allocations.each do |allocation|
+        ids = allocation.process! amount, offset
+        @total = @total + allocation.count # the total mixed in
+        if ids.empty?
+          offset = offset - allocation.count unless offset.zero?
+        else
+          amount = amount - ids.size # we need less results from the following allocation
+          offset = 0                 # we have already passed the offset
+        end
+      end
+    end
+
+    def uniq
+      @allocations.uniq!
+    end
+
+    def to_a
+      @allocations
+    end
+
+    # Simply inspects the internal allocations.
+    #
+    def to_s
+      @allocations.inspect
+    end
+
+    # Allocations for results are in the form:
+    # [
+    #   allocation1.to_result,
+    #   allocation2.to_result
+    #   ...
+    # ]
+    #
+    def to_result
+      @allocations.map(&:to_result).compact
+    end
+
+  end
+
+end

data/lib/picky/query/combination.rb

@@ -0,0 +1,76 @@
+module Query
+
+  # Describes the combination of a token (the text) and
+  # the index (the bundle): [text, index_bundle]
+  #
+  # A combination is a single part of an allocation:
+  # [..., [text2, index_bundle2], ...]
+  #
+  # An allocation consists of a number of combinations:
+  # [[text1, index_bundle1], [text2, index_bundle2], [text3, index_bundle1]]
+  #
+  class Combination # :nodoc:all
+
+    attr_reader :token, :bundle, :category_name
+
+    def initialize token, category
+      @token         = token
+      @category_name = category.name
+      @bundle        = category.bundle_for token
+      @text          = @token.text # don't want to use reset_similar already
+    end
+  
+    # Note: Required for uniq!
+    #
+    def hash
+      [@token.to_s, @bundle].hash
+    end
+  
+    # Returns the weight of this combination.
+    #
+    # Note: Caching is most of the time useful.
+    #
+    def weight
+      @weight ||= @bundle.weight(@text)
+    end
+  
+    # Returns an array of ids for the given text.
+    #
+    # Note: Caching is most of the time useful.
+    #
+    def ids
+      @ids ||= @bundle.ids(@text)
+    end
+  
+    # The identifier for this combination.
+    #
+    def identifier
+      "#{bundle.identifier}:#{@token.identifier}"
+    end
+  
+    # Is the identifier in the given identifiers?
+    #
+    def in? identifiers
+      identifiers.include? identifier
+    end
+
+    # Combines the category names with the original names.
+    # [
+    #  [:title,    'Flarbl', :flarbl],
+    #  [:category, 'Gnorf',  :gnorf]
+    # ]
+    #
+    def to_result
+      [@category_name, *@token.to_result]
+    end
+  
+    # Example:
+    #  "exact title:Peter*:peter"
+    #
+    def to_s
+      "#{bundle.identifier} #{to_result.join(':')}"
+    end
+  
+  end
+
+end