picky 0.0.0 → 0.0.2

data/lib/picky/query/combination.rb

@@ -0,0 +1,69 @@
+module Query
+
+  # Describes the combination of a token (the text) and
+  # the index (the bundle).
+  #
+  # A combination is a single part of an allocation.
+  #
+  # An allocation consists of a number of combinations.
+  #
+  class Combination
+
+    attr_reader :token, :bundle
+
+    def initialize token, category
+      @token    = token
+      @category = category
+      @bundle   = category.bundle_for token
+      @text     = @token.text # don't want to use reset_similar already
+    end
+    
+    # Note: Required for uniq!
+    #
+    def hash
+      [@token.to_s, @bundle].hash
+    end
+    
+    # Returns the weight of this combination.
+    #
+    def weight
+      @weight || @weight = @bundle.weight(@text)
+    end
+    
+    # Returns an array of ids for the given text.
+    #
+    def ids
+      @ids || @ids = @bundle.ids(@text)
+    end
+    
+    # The identifier for this combination.
+    #
+    def identifier
+      @category.name
+    end
+    
+    # Is the identifier in the given identifiers?
+    #
+    def in? identifiers
+      identifiers.include? identifier
+    end
+
+    # Combines the category names with the original names.
+    # [
+    #  [:title,    'Flarbl', :flarbl],
+    #  [:category, 'Gnorf',  :gnorf]
+    # ]
+    #
+    def to_result
+      [identifier, *@token.to_result]
+    end
+    
+    # full/title:Flarbl:flarbl
+    #
+    def to_s
+      "#{bundle.name}/#{to_result.join(':')}"
+    end
+    
+  end
+
+end

data/lib/picky/query/combinations.rb

@@ -0,0 +1,106 @@
+module Query
+
+  # Combinations are a number of Combination-s.
+  #
+  # They are, in effect, the core of an allocation.
+  #
+  class Combinations
+
+    attr_reader :type, :combinations
+
+    delegate :empty?, :to => :@combinations
+
+    def initialize type, combinations = []
+      @type         = type # TODO Remove.
+      @combinations = combinations
+    end
+
+    def hash
+      @combinations.hash
+    end
+
+    #
+    #
+    # TODO Rewrite.
+    #
+    def calculate_score weights
+      @score  = @combinations.sum &:weight
+      @score += weights.score @combinations
+      @score
+    end
+
+    # Gets all ids for the allocations.
+    #
+    # Sorts the ids by size and & through them in the following order (sizes):
+    # 0. [100_000, 400, 30, 2]
+    # 1. [2, 30, 400, 100_000]
+    # 2. (100_000 & (400 & (30 & 2))) # => result
+    #
+    # Returns the ids. Also sets the count.
+    #
+    # Note: Uses a C-optimized intersection routine for speed and memory efficiency.
+    #
+    def ids
+      return [] if @combinations.empty?
+
+      # Get the ids for each combination.
+      #
+      id_arrays = @combinations.inject([]) do |total, combination|
+        total << combination.ids
+      end
+
+      # Order by smallest size first such that the intersect can be performed faster.
+      #
+      # TODO Move into the memory_efficient_intersect such that
+      #      this precondition for a fast algorithm is always given.
+      #
+      id_arrays.sort! { |this_array, that_array| this_array.size <=> that_array.size }
+
+      # Call the optimized C algorithm.
+      #
+      Performant::Array.memory_efficient_intersect id_arrays
+    end
+
+    #
+    #
+    def pack_into_allocation
+      allocation = Allocation.new self
+      allocation.result_type = @type.result_type # TODO Rewrite.
+      allocation
+    end
+
+    # Filters the tokens and identifiers such that only identifiers
+    # that are passed in, remain, including their tokens.
+    #
+    # Note: This method is not totally independent of the calculate_ids one.
+    #       Since identifiers are only nullified, we need to not include the
+    #       ids that have an associated identifier that is nil.
+    #
+    def keep identifiers = []
+      # TODO Rewrite to use the category!!!
+      #
+      @combinations.reject! { |combination| !combination.in?(identifiers) }
+    end
+
+    # Filters the tokens and identifiers such that identifiers
+    # that are passed in, are removed, including their tokens.
+    #
+    # Note: This method is not totally independent of the calculate_ids one.
+    #       Since identifiers are only nullified, we need to not include the
+    #       ids that have an associated identifier that is nil.
+    #
+    def remove identifiers = []
+      # TODO Rewrite to use the category!!!
+      #
+      @combinations.reject! { |combination| combination.in?(identifiers) }
+    end
+
+    #
+    #
+    def to_result
+      @combinations.map &:to_result
+    end
+
+  end
+
+end

data/lib/picky/query/combinator.rb

@@ -0,0 +1,92 @@
+module Query
+  
+  # Combines tokens and category indexes into combinations.
+  #
+  class Combinator
+    
+    attr_reader :categories, :category_hash
+    attr_reader :ignore_unassigned_tokens # TODO Should this actually be determined by the query? Probably, yes.
+    
+    def initialize categories, options = {}
+      @categories               = categories
+      @category_hash            = hashify categories
+      
+      @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
+    end
+    
+    # TODO Move somewhere else.
+    #
+    # TODO Or use active_support's?
+    #
+    def hashify category_array
+      category_array.inject({}) do |hash, category|
+        hash[category.name] = [category]
+        hash
+      end
+    end
+    
+    #
+    #
+    def possible_combinations_for token
+      token.similar? ? similar_possible_for(token) : possible_for(token)
+    end
+    
+    # TODO Break apart.
+    #
+    def similar_possible_for token
+      # Get as many similar tokens as necessary
+      #
+      text = token.text
+      tokens = categories.inject([]) do |result, category|
+        next_token = token
+        # TODO adjust either this or the amount of similar in index
+        #
+        while next_token = next_token.next(category)
+          result << next_token if next_token && next_token.text != text
+        end
+        result
+      end
+      # possible combinations
+      #
+      tokens.inject([]) do |result, token|
+        possible = possible_categories token
+        result + possible_for(token, possible)
+      end
+    end
+    
+    # Returns possible Combinations for the token.
+    #
+    # The categories param is an optimization.
+    #
+    # TODO Return [RemovedCategory(token, nil)]
+    #      If the search is ...
+    #
+    # TODO Make categories also a collection class.
+    #
+    # TODO Return [] if not ok, nil if needs to be removed?
+    #      Somehow unnice, but…
+    #
+    def possible_for token, preselected_categories = nil
+      possible = (preselected_categories || possible_categories(token)).map { |category| category.combination_for(token) }
+      possible.compact!
+      # This is an optimization to mark tokens that are ignored.
+      #
+      return if ignore_unassigned_tokens && possible.empty?
+      possible # wrap in combinations
+    end
+    #
+    #
+    # TODO too many calls?
+    #
+    def possible_categories token
+      user_defined_categories(token) || categories
+    end
+    # Returns nil if there is no user defined category, the category else.
+    #
+    def user_defined_categories token
+      category_hash[token.user_defined_category_name]
+    end
+    
+  end
+  
+end

data/lib/picky/query/full.rb

@@ -0,0 +1,15 @@
+module Query
+
+  # This is the query class for performing full fledged queries.
+  #
+  class Full < Base
+
+    # Generates full results.
+    #
+    def result_type
+      Results::Full
+    end
+
+  end
+
+end

data/lib/picky/query/live.rb

@@ -0,0 +1,22 @@
+module Query
+
+  # This is the query class for live queries.
+  #
+  # It does:
+  #  * Return a count of results.
+  #
+  # It does NOT:
+  #  * Sort results geographically.
+  #  * Do any postprocessing.
+  #
+  class Live < Base
+
+    # Generates results from allocations.
+    #
+    def result_type
+      Results::Live
+    end
+
+  end
+
+end

data/lib/picky/query/qualifiers.rb

@@ -0,0 +1,73 @@
+# coding: utf-8
+module Query
+
+  # A single qualifier.
+  #
+  class Qualifier
+
+    attr_reader :normalized_qualifier, :codes
+
+    #
+    #
+    # codes is an array.
+    #
+    def initialize normalized_qualifier, codes
+      @normalized_qualifier = normalized_qualifier
+      @codes                = codes
+    end
+
+    # Will overwrite if the key is present in the hash.
+    #
+    def inject_into hash
+      codes.each do |code|
+        hash[code] = normalized_qualifier
+      end
+    end
+
+  end
+
+  # Collection class for qualifiers.
+  #
+  class Qualifiers
+
+    include Singleton
+
+    attr_reader :qualifiers, :normalization_mapping
+
+    delegate :<<, :to => :qualifiers
+
+    #
+    #
+    def initialize
+      @qualifiers = []
+      @normalization_mapping = {}
+    end
+    
+    # TODO Spec.
+    #
+    def self.add name, qualifiers
+      instance << Qualifier.new(name, qualifiers)
+    end
+    
+    # Uses the qualifiers to prepare (optimize) the qualifier handling.
+    #
+    def prepare
+      qualifiers.each do |qualifier|
+        qualifier.inject_into normalization_mapping
+      end
+    end
+
+    # Normalizes the given qualifier.
+    #
+    # Returns nil if it is not allowed, the normalized qualifier if it is.
+    #
+    # Note: Normalizes.
+    #
+    def normalize qualifier
+      return nil if qualifier.blank?
+
+      normalization_mapping[qualifier.to_sym]
+    end
+
+  end
+end

data/lib/picky/query/solr.rb

@@ -0,0 +1,77 @@
+require 'rsolr'
+
+module Query
+
+  #
+  #
+  class Solr < Base
+
+    attr_reader :server, :index_types
+
+    def initialize *index_types
+      @server = RSolr.connect rescue nil
+      super *index_types
+    end
+
+    # # This runs the actual search.
+    # #
+    # # TODO Remove!
+    # #
+    # def search tokens, offset = 0
+    #   results = nil
+    #
+    #   duration = timed do
+    #     results = execute(tokens, offset) || empty_results # TODO Does not work yet
+    #   end
+    #   results.duration = duration
+    #
+    #   results
+    # end
+
+    #
+    #
+    def execute tokens, offset = 0
+      results = Results::Live.new
+
+      if server
+        similar = {}
+
+        new_query = tokens.to_solr_query
+
+        return results if new_query.empty?
+
+        index_types.each do |index|
+          begin
+            response = server.select :q => new_query, :fq => "type:#{index.name}", :hl => true, :'hl.fl' => '*', :'hl.simple.pre' => '<', :'hl.simple.post' => '>', :facet => true
+          rescue RSolr::RequestError => re
+            return results
+          end
+
+          highlighting = response['highlighting']
+          possibilities = response['response']['docs'].map do |doc|
+            highlights = highlighting[doc['id'].to_s]
+            next unless highlights
+            selected = doc.select { |key| highlights.has_key?(key) }
+            selected.values.join ' '
+          end
+          possibilities.collect! { |possibility| possibility.strip }.uniq!
+          similar[index.name] = possibilities unless possibilities.empty?
+        end
+
+        results.add :similar => similar
+      end
+
+      # TODO
+      #
+      class << results
+        def to_log query
+          ?* + super
+        end
+      end
+
+      results
+    end
+
+  end
+
+end

data/lib/picky/query/token.rb

@@ -0,0 +1,215 @@
+module Query
+  # This is a query token. Together with other tokens it makes up a query.
+  #
+  # It remembers the original form, and and a normalized form.
+  #
+  # It also knows whether it needs to look for similarity (bla~), or whether it is a partial (bla*).
+  #
+  # TODO Make partial / similarity char configurable.
+  #
+  class Token
+
+    attr_reader :text, :original
+    attr_writer :similar
+
+    delegate :blank?, :to => :text
+
+    # Normal initializer.
+    #
+    # Note: Use this if you do not want a qualified and normalized token.
+    #
+    def initialize text
+      @text = text
+    end
+
+    # Returns a qualified and normalized token.
+    #
+    # Note: Use this in the search engine if you need a qualified
+    #       and normalized token. I.e. one prepared for a search.
+    #
+    def self.processed text
+      token = new text
+      token.qualify
+      token.extract_original
+      token.partialize
+      token.similarize
+      token.remove_illegals
+      token
+    end
+
+    # This returns a predefined category name if the user has given one.
+    #
+    def user_defined_category_name
+      @qualifier
+    end
+
+    # Extracts a qualifier for this token and pre-assigns an allocation.
+    #
+    # Note: Removes the qualifier if it is not allowed.
+    #
+    def qualify
+      @qualifier, @text = split @text
+      @qualifier = Query::Qualifiers.instance.normalize @qualifier
+    end
+    def extract_original
+      @original = @text.dup
+    end
+
+    # Partial is a conditional setter.
+    #
+    # It is only settable if it hasn't been set yet.
+    #
+    def partial= partial
+      @partial = partial if @partial.nil?
+    end
+    def partial?
+      !@similar && @partial
+    end
+
+    # If the text ends with *, partialize it. If with ", don't.
+    #
+    @@no_partial = /\"$/
+    @@partial    = /[\*]$/
+    def partialize
+      self.partial = false and return if @text =~ @@no_partial
+      self.partial = true if @text =~ @@partial
+    end
+
+    # If the text ends with ~ similarize it. If with ", don't.
+    #
+    @@no_similar = /\"$/
+    @@similar    = /[~]$/
+    def similarize
+      self.similar = false and return if @text =~ @@no_similar
+      self.similar = true if @text =~ @@similar
+    end
+
+    def similar?
+      @similar
+    end
+
+    # Normalizes this token's text.
+    #
+    @@illegals = /["*~]/
+    def remove_illegals
+      @text.gsub! @@illegals, '' unless @text.blank?
+    end
+
+    # TODO Think about these, remove illegals and normalize...
+    #
+
+    # Visitor for tokenizer.
+    #
+    # TODO Rewrite!!!
+    #
+    def tokenize_with tokenizer
+      @text = tokenizer.normalize @text
+    end
+    # TODO spec!
+    #
+    # TODO Rewrite!!
+    #
+    def tokenized tokenizer
+      tokenizer.tokenize(@text.to_s).each do |text|
+        yield text
+      end
+    end
+
+    # Returns an array of possible combinations.
+    #
+    def possible_combinations_in type
+      type.possible_combinations self
+    end
+
+    #
+    #
+    def from token
+      new_token = token.dup
+      new_token.instance_variable_set :@text, @text
+      new_token.instance_variable_set :@partial, @partial
+      new_token.instance_variable_set :@original, @original
+      new_token.instance_variable_set :@qualifier, @qualifier
+      # TODO
+      #
+      # token.instance_variable_set :@similarity, @similarity
+      new_token
+    end
+
+    # TODO Rewrite, also next_similar.
+    #
+    def next category
+      token = from self
+      token if token.next_similar category.bundle_for(token)
+    end
+
+    # Sets and returns the next similar word.
+    #
+    def next_similar bundle
+      @text = similarity(bundle).next if similar?
+    rescue StopIteration => stop_iteration
+      # reset_similar # TODO
+      nil # TODO
+    end
+    # Lazy similar reader.
+    #
+    def similarity bundle = nil
+      @similarity || @similarity = generate_similarity_for(bundle)
+    end
+    # Returns an enumerator that traverses over the similar.
+    #
+    def generate_similarity_for bundle
+      (bundle.similar(@text) || []).each
+    end
+
+    # Generates a solr term from this token.
+    #
+    # E.g. "name:heroes~0.75"
+    #
+    @@solr_fuzzy_mapping = {
+      1 => :'',
+      2 => :'',
+      3 => :'',
+      4 => :'~0.74',
+      5 => :'~0.78',
+      6 => :'~0.81',
+      7 => :'~0.83',
+      8 => :'~0.85',
+      9 => :'~0.87',
+     10 => :'~0.89'
+    }
+    @@solr_fuzzy_mapping.default = :'~0.9'
+    def to_solr
+      blank? ? '' : (to_s + @@solr_fuzzy_mapping[@text.size].to_s)
+    end
+
+    #
+    #
+    def to_result
+      [@original, @text]
+    end
+
+    # Displays the qualifier text and the text, joined.
+    #
+    # e.g. name:meier
+    #
+    def to_s
+      [@qualifier, @text].compact.join ':'
+    end
+
+    private
+
+      # Splits text into a qualifier and text.
+      #
+      # Returns [qualifier, text].
+      #
+      def split unqualified_text
+        qualifier, text = (unqualified_text || '').split(':', 2)
+        if text.blank?
+          [nil, (qualifier || '')]
+        else
+          [qualifier, text]
+        end
+      end
+
+  end
+end