picky 0.0.0 → 0.0.2

data/lib/picky/loader.rb

@@ -0,0 +1,287 @@
+# Loads the search engine and itself.
+#
+module Loader
+  
+  # Reloads the whole app.
+  # First itself, then the app.
+  #
+  def self.reload
+    Dir.chdir(SEARCH_ROOT)
+    exclaim 'Reloading loader.'
+    load_self
+    exclaim 'Reloading framework.'
+    load_framework
+    exclaim 'Reloading application.'
+    load_application
+  end
+
+  # Loads this file anew.
+  #
+  def self.load_self
+    exclaim 'Loader loading itself.'
+    load __FILE__
+  end
+  
+  def self.require_relative filename
+    require File.join(File.dirname(__FILE__), filename)
+  end
+  def self.load_relative filename_without_rb
+    load File.join(File.dirname(__FILE__), "#{filename_without_rb}.rb")
+  end
+  
+  def self.load_user filename
+    load File.join(SEARCH_ROOT, "#{filename}.rb")
+  end
+  def self.load_user_lib filename
+    load_user File.join('lib', filename)
+  end
+  def self.load_all_user_in dirname
+    Dir[File.join(SEARCH_ROOT, dirname, '**', '*.rb')].each do |filename|
+      load filename
+    end
+  end
+  
+  # def self.add_lib_dir
+  #   lib_dir = File.join(SEARCH_ROOT, 'lib')
+  #   $:.unshift lib_dir unless $:.include?(lib_dir)
+  # end
+  
+  # Load the user's application.
+  #
+  def self.load_application
+    # DB.connect # FIXME only needed when indexing.
+    # Load the user's application.
+    #
+    exclaim 'Loading Application.'
+    
+    load_all_user_in 'app/initializers'
+    
+    # Add lib dir to load path.
+    #
+    # add_lib_dir
+    
+    # Picky autoloading.
+    #
+    begin
+      load_all_user_in 'lib/tokenizers'
+      load_all_user_in 'lib/indexers'
+      load_all_user_in 'lib/query'
+    rescue NameError => name_error
+      namespaced_class_name = name_error.message.gsub /^uninitialized\sconstant\s/, ''
+      load_user_lib namespaced_class_name.underscore # Try it once.
+      retry
+    end
+    
+    # TODO Rethink this.
+    #
+    load_user 'app/logging'
+    # load_user 'app/config'
+    # Configuration.apply
+    
+    # Require the user's application.
+    #
+    load_user 'app/application'
+    
+    # Setup Indexes from user definition
+    #
+    # TODO Make special task that reloads the indexes!
+    #
+    Query::Qualifiers.instance.prepare # TODO Rewrite
+    
+    exclaim "Application loaded."
+  end
+  
+  # Loads the framework.
+  #
+  def self.load_framework
+    # exclaim 'Loading the application.'
+    
+    # exclaim 'Compiling C code.'
+    require_relative 'initializers/ext'
+    
+    require 'rack_fast_escape'
+    # exclaim 'Loaded rack_fast_escape.'
+    require 'text'
+    # exclaim 'Loaded text.'
+    
+    # Extend path with lib
+    #
+    extend_load_path 'lib'
+    
+    # Load extensions.
+    #
+    load_relative 'extensions/array'
+    load_relative 'extensions/symbol'
+    load_relative 'extensions/module'
+    load_relative 'extensions/hash'
+    # exclaim "Loaded extensions."
+    
+    # Load harakiri.
+    #
+    load_relative 'rack/harakiri'
+    
+    # Requiring Helpers
+    #
+    load_relative 'helpers/gc'
+    load_relative 'helpers/cache'
+    load_relative 'helpers/measuring'
+    load_relative 'helpers/search'
+    # exclaim "Loaded helpers."
+    
+    # Signal handling
+    #
+    load_relative 'signals'
+    # exclaim "Loaded signals handling."
+
+    # Load and require the plugins.
+    #
+    Dir['plugins/*'].each do |directory|
+      extend_load_path directory
+      extend_load_path directory, 'lib'
+      load "#{directory.gsub!(/plugins\//, '')}.rb"
+    end
+    # exclaim "Loaded plugins."
+
+    # Require the necessary libs. Referenced modules first.
+    #
+    load_relative 'loggers/search'
+    load_relative 'umlaut_substituter'
+
+    # Index generation strategies.
+    #
+    load_relative 'indexers/no_source_specified_error'
+    load_relative 'indexers/base'
+    load_relative 'indexers/field'
+    load_relative 'indexers/default'
+    #
+    load_relative 'indexers/solr'
+
+    # Partial index generation strategies.
+    #
+    load_relative 'cacher/partial/strategy'
+    load_relative 'cacher/partial/none'
+    load_relative 'cacher/partial/subtoken'
+    load_relative 'cacher/partial/default'
+
+    # Weight index generation strategies.
+    #
+    load_relative 'cacher/weights/strategy'
+    load_relative 'cacher/weights/logarithmic'
+    load_relative 'cacher/weights/default'
+    
+    # Similarity index generation strategies.
+    #
+    load_relative 'cacher/similarity/strategy'
+    load_relative 'cacher/similarity/none'
+    load_relative 'cacher/similarity/double_levenshtone'
+    load_relative 'cacher/similarity/default'
+    
+    # Convenience accessors for generators.
+    #
+    # TODO Just remove from under Cacher?
+    #
+    load_relative 'cacher/convenience'
+    
+    # Index generators.
+    #
+    load_relative 'cacher/generator'
+    load_relative 'cacher/partial_generator'
+    load_relative 'cacher/weights_generator'
+    load_relative 'cacher/similarity_generator'
+    
+    # Index types.
+    #
+    load_relative 'index/bundle'
+    load_relative 'index/category'
+    load_relative 'index/type'
+    
+    load_relative 'index/combined'
+    
+    # Tokens.
+    #
+    load_relative 'query/token'
+    load_relative 'query/tokens'
+    
+    # Tokenizers types.
+    #
+    load_relative 'tokenizers/base'
+    load_relative 'tokenizers/index'
+    load_relative 'tokenizers/query'
+    load_relative 'tokenizers/default'
+    
+    # Query combinations, qualifiers, weigher.
+    #
+    load_relative 'query/combination'
+    load_relative 'query/combinations'
+
+    load_relative 'query/allocation'
+    load_relative 'query/allocations'
+
+    load_relative 'query/qualifiers'
+    load_relative 'query/weigher'
+    load_relative 'query/combinator'
+
+    load_relative 'query/weights'
+
+    # Query.
+    #
+    load_relative 'query/base'
+    load_relative 'query/live'
+    load_relative 'query/full'
+    load_relative 'query/solr' # TODO ?
+
+    # Results.
+    #
+    load_relative 'results/base'
+    load_relative 'results/full'
+    load_relative 'results/live'
+    
+    # Sources.
+    #
+    load_relative 'sources/base'
+    load_relative 'sources/db'
+    
+    # DB
+    #
+    load_relative 'db/configuration'
+    
+    # Indexes.
+    #
+    load_relative 'indexes'
+    
+    # Configuration.
+    #
+    load_relative 'configuration/field'
+    load_relative 'configuration/type'
+    load_relative 'configuration/indexes'
+    load_relative 'configuration/configuration'
+    
+    # ... in Application.
+    #
+    load_relative 'configuration/queries'
+    
+    # Application and routing.
+    #
+    load_relative 'routing'
+    load_relative 'application'
+    
+    # Load tools.
+    #
+    load_relative 'solr/schema_generator'
+    load_relative 'cores'
+    
+    # Load generation.
+    #
+    load_relative 'generator'
+  end
+
+  def self.exclaim text
+    puts text
+  end
+
+  def self.extend_load_path *dirs
+    dir = File.join(SEARCH_ROOT, *dirs)
+    $:.unshift dir unless $:.include? dir
+  end
+
+end

data/lib/picky/loggers/search.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+#
+module Loggers
+  # Log Proxy
+  #
+  class Search
+    
+    attr_reader :logger
+    
+    def initialize logger
+      @logger = logger
+    end
+    
+    def log message
+      logger.info message
+    end
+    
+  end
+end

data/lib/picky/performant/array.rb

@@ -0,0 +1,23 @@
+module Performant
+  # This class will be enriched with c-methods
+  #
+  class Array
+
+    # Chooses a good algorithm for intersecting arrays.
+    #
+    # Note: The sort order will be changed.
+    #
+    def self.intersect array_of_arrays
+      array_of_arrays.sort! { |a, b| a.size <=> b.size }
+
+      if (array_of_arrays.sum(&:size) < 20_000)
+        Performant::Array.brute_force_intersect array_of_arrays
+      else
+        array_of_arrays.inject([]) do |total, elements|
+          total.empty? ? elements : elements & total
+        end
+      end
+    end
+
+  end
+end

data/lib/picky/query/allocation.rb

@@ -0,0 +1,82 @@
+module Query
+  # An allocation has a number of combinations:
+  # [token, index] [other_token, other_index], ...
+  #
+  class Allocation
+
+    attr_reader   :count, :ids, :score, :combinations
+    attr_accessor :result_type
+
+    #
+    #
+    def initialize combinations
+      @combinations = combinations
+    end
+
+    def hash
+      @combinations.hash
+    end
+    def eql? other_allocation
+      true # FIXME
+      # @combinations.eql? other_allocation.combinations
+    end
+
+    # Scores its combinations and caches the result.
+    #
+    def calculate_score weights
+      @score || @score = @combinations.calculate_score(weights)
+    end
+
+    # Asks the combinations for the (intersected) ids.
+    #
+    def calculate_ids
+      @combinations.ids
+    end
+
+    # This starts the searching process.
+    #
+    def process! amount, offset
+      ids    = calculate_ids
+      @count = ids.size                         # cache the count before throwing away the ids
+      @ids   = ids.slice!(offset, amount) || [] # slice out the relevant part
+    end
+
+    #
+    #
+    def keep identifiers = [] # categories
+      @combinations.keep identifiers
+    end
+    #
+    #
+    def remove identifiers = [] # categories
+      @combinations.remove identifiers
+    end
+
+    # Sort highest score first.
+    #
+    def <=> other_allocation
+       other_allocation.score <=> self.score
+    end
+
+    # Transform the allocation into result form.
+    #
+    def to_result
+      [self.result_type, self.score, self.count, @combinations.to_result, self.ids] if self.count > 0
+    end
+
+    # Json representation of this allocation.
+    #
+    # Note: Delegates to to_result.
+    #
+    def to_json
+      to_result.to_json
+    end
+
+    #
+    #
+    def to_s
+      "Allocation: #{to_result.join(', ')}"
+    end
+
+  end
+end

data/lib/picky/query/allocations.rb

@@ -0,0 +1,131 @@
+module Query
+  # Container class for allocations.
+  #
+  class Allocations
+
+    # TODO Remove size
+    #
+    delegate :each, :inject, :empty?, :size, :to => :@allocations
+    attr_reader :total
+
+    def initialize allocations = []
+      @allocations = allocations
+    end
+
+    # Score each allocation.
+    #
+    def calculate_score weights
+      @allocations.each do |allocation|
+        allocation.calculate_score weights
+      end
+    end
+    # Sort the allocations.
+    #
+    def sort
+      @allocations.sort!
+    end
+
+    # Reduces the amount of allocations to x.
+    #
+    def reduce_to amount
+      @allocations = @allocations.shift amount
+    end
+
+    # Keeps combinations.
+    #
+    # Only those passed in remain.
+    #
+    def keep identifiers = []
+      @allocations.each { |allocation| allocation.keep identifiers } unless identifiers.empty?
+    end
+    # Removes combinations.
+    #
+    # Only those passed in are removed.
+    #
+    # TODO Rewrite
+    #
+    def remove identifiers = []
+      @allocations.each { |allocation| allocation.remove identifiers } unless identifiers.empty?
+    end
+
+    # Returns the top amount ids.
+    #
+    def ids amount = 20
+      @allocations.inject([]) do |total, allocation|
+        total.size >= amount ? (return total.shift(amount)) : total + allocation.ids
+      end
+    end
+
+    # Returns a random id from the allocations.
+    #
+    # Note: This is an ok algorithm for small id sets.
+    #
+    # But still TODO try for a faster one.
+    #
+    def random_ids amount = 1
+      # TODO can there be no @allocations???
+      return [] if @allocations.empty?
+      ids = @allocations.first.ids
+      indexes = Array.new(ids.size) { |i| i }.sort_by { rand }
+      indexes.first(amount).map { |i| ids[i] }
+    end
+
+    # This is the main method of this class that will replace ids and count.
+    #
+    # What it does is calculate the ids and counts of its allocations
+    # for being used in the results. It also calculates the total
+    #
+    # Parameters:
+    #  * amount: the amount of ids to calculate
+    #  * offset: the offset from where in the result set to take the ids
+    #
+    # Note: With an amount of 0, an offset > 0 doesn't make much
+    #       sense, as seen in the live search.
+    #
+    # Note: Each allocation caches its count, but not its ids (thrown away).
+    #       The ids are cached in this class.
+    #
+    # Note: It's possible that no ids are returned by an allocation, but a count. (In case of an offset)
+    #
+    def process! amount, offset = 0
+      @total = 0
+      current_offset = 0
+      @allocations.each do |allocation|
+        ids = allocation.process! amount, offset
+        @total = @total + allocation.count # the total mixed in
+        if ids.empty?
+          offset = offset - allocation.count unless offset.zero?
+        else
+          amount = amount - ids.size # we need less results from the following allocation
+          offset = 0                 # we have already passed the offset
+        end
+      end
+    end
+
+    def uniq
+      @allocations.uniq!
+    end
+
+    def to_a
+      @allocations
+    end
+
+    # Simply inspects the internal allocations.
+    #
+    def to_s
+      @allocations.inspect
+    end
+
+    # Allocations for results are in the form:
+    # [
+    #   allocation1.to_result,
+    #   allocation2.to_result
+    #   ...
+    # ]
+    #
+    def to_result
+      @allocations.map(&:to_result).compact
+    end
+
+  end
+end

data/lib/picky/query/base.rb

@@ -0,0 +1,124 @@
+module Query
+  # Base query class.
+  #
+  # Initialized with the index types it should search on.
+  #
+  class Base
+    
+    include Helpers::Measuring
+    
+    attr_writer   :tokenizer
+    attr_accessor :reduce_to_amount, :weights
+    
+    # Run a query on the given text, with the offset and these indexes.
+    #
+    def initialize *index_types
+      options      = Hash === index_types.last ? index_types.pop : {}
+      @index_types = index_types
+      @weigher     = Weigher.new index_types
+      @tokenizer   = (options[:tokenizer]  || Tokenizers::Query.new)
+      @weights     = (options[:weights] || Weights.new)
+    end
+    
+    # Convenience method.
+    #
+    def search_with_text text, offset = 0
+      search tokenized(text), offset
+    end
+    
+    # This runs the actual search.
+    #
+    def search tokens, offset = 0
+      results = nil
+      
+      duration = timed do
+        results = execute(tokens, offset) || empty_results # TODO Does not work yet
+      end
+      results.duration = duration.round 6
+      
+      results
+    end
+    
+    # Return nil if no results have been found.
+    #
+    def execute tokens, offset
+      results_from sorted_allocations(tokens), offset
+    end
+
+    # Returns an empty result with default values.
+    #
+    def empty_results
+      result_type.new
+    end
+
+    # Delegates the tokenizing to the query tokenizer.
+    #
+    def tokenized text
+      @tokenizer.tokenize text
+    end
+
+    # Gets sorted allocations for the tokens.
+    #
+    # This generates the possible allocations, sorted.
+    #
+    # TODO Smallify.
+    #
+    def sorted_allocations tokens
+      # Get the allocations.
+      #
+      allocations = @weigher.allocations_for tokens
+
+      # Callbacks.
+      #
+      reduce allocations
+      remove_from allocations
+
+      # TODO allocations#calculate # or better name
+      #
+
+      # Remove double allocations.
+      #
+      allocations.uniq
+
+      # Score the allocations.
+      #
+      allocations.calculate_score weights
+
+      # Sort the allocations.
+      # (allocations are sorted according to score, highest to lowest)
+      #
+      allocations.sort
+
+      # Return the allocations.
+      #
+      allocations
+    end
+    def reduce allocations
+      allocations.reduce_to reduce_to_amount if reduce_to_amount
+    end
+    def remove_identifiers?
+      identifiers_to_remove.present?
+    end
+    def remove_from allocations
+      allocations.remove(identifiers_to_remove) if remove_identifiers?
+    end
+    # Override.
+    #
+    def identifiers_to_remove
+      @identifiers_to_remove || @identifiers_to_remove = []
+    end
+
+    # Packs the sorted allocations into results.
+    #
+    # This generates the id intersections. Lots of work going on.
+    #
+    # TODO Move to results. result_type.from allocations, offset
+    #
+    def results_from allocations = nil, offset = 0
+      results = result_type.new allocations
+      results.prepare! offset
+      results
+    end
+
+  end
+end