picky 0.3.0 → 0.9.0

data/lib/picky/application.rb

@@ -34,7 +34,7 @@ class Application
     #
     delegate :route, :root, :to => :routing
     
-    # 
+    # TODO Rename to default_indexing?
     #
     def indexing
       @indexing ||= Configuration::Indexes.new
@@ -44,7 +44,7 @@ class Application
     end
     delegate :type, :field, :to => :indexing
     
-    #
+    # TODO Rename to default_querying?
     #
     def querying
       @queries ||= Configuration::Queries.new

data/lib/picky/cacher/partial/default.rb

@@ -1,5 +1,5 @@
 module Cacher
   module Partial
-    Default = Subtoken.new :down_to => 1, :starting_at => -1
+    Default = Subtoken.new :down_to => -3, :starting_at => -1
   end
 end

data/lib/picky/configuration/field.rb

@@ -4,19 +4,20 @@ module Configuration
   # (title is a category of a books index, for example).
   #
   class Field
-    attr_reader :name, :indexed_name, :virtual
+    attr_reader :name, :indexed_name, :virtual, :tokenizer
     attr_accessor :type # convenience
-    def initialize name, options = {}
+    def initialize name, tokenizer, options = {}
       @name            = name.to_sym
+      @tokenizer       = tokenizer
       
       # TODO Dup the options?
-      # TODO add source as option
+      
+      @source          = options.delete :source
       
       @indexer_class   = options.delete(:indexer)   || Indexers::Default
-      @tokenizer_class = options.delete(:tokenizer) || Tokenizers::Index # Default
       
-      @indexed_name   = options.delete(:indexed_field) || name # TODO Rename to indexed_as?
-      @virtual        = options.delete(:virtual)       || false
+      @indexed_name    = options.delete(:indexed_field) || name # TODO Rename to indexed_as?
+      @virtual         = options.delete(:virtual)       || false
       
       qualifiers = generate_qualifiers_from options
       Query::Qualifiers.add(name, qualifiers) if qualifiers
@@ -48,7 +49,7 @@ module Configuration
       File.join search_index_root, PICKY_ENVIRONMENT, type.name.to_s
     end
     def search_index_file_name
-      File.join cache_directory, "#{type.name}_#{name}_index.txt"
+      File.join cache_directory, "prepared_#{name}_index.txt"
     end
     def index
       prepare_cache_directory
@@ -64,9 +65,6 @@ module Configuration
     def indexer
       @indexer || @indexer = @indexer_class.new(type, self)
     end
-    def tokenizer
-      @tokenizer || @tokenizer = @tokenizer_class.new
-    end
     def virtual?
       !!virtual
     end

data/lib/picky/configuration/indexes.rb

@@ -10,15 +10,13 @@ module Configuration
       @types = []
     end
     
-    #
-    #
-    def default_index
-      Tokenizers::Index
+    def default_tokenizer
+      @default_tokenizer ||= Tokenizers::Default::Index
     end
     
     # Delegates
     #
-    delegate :removes_characters, :contracts_expressions, :stopwords, :splits_text_on, :normalizes_words, :removes_characters_after_splitting, :to => :default_index
+    delegate :removes_characters, :contracts_expressions, :stopwords, :splits_text_on, :normalizes_words, :removes_characters_after_splitting, :to => :default_tokenizer
     
     # TODO Rewrite all this configuration handling.
     #
@@ -32,7 +30,9 @@ module Configuration
       generated
     end
     def field name, options = {}
-      Field.new name, options
+      tokenizer = options[:tokenizer] || default_tokenizer
+      
+      Field.new name, tokenizer, options
     end
     
     #

data/lib/picky/configuration/queries.rb

@@ -6,10 +6,11 @@ module Configuration
     
     #
     #
-    def default_index
-      Tokenizers::Query
+    def default_tokenizer
+      @default_tokenizer ||= Tokenizers::Default::Query
     end
-    delegate :removes_characters, :contracts_expressions, :stopwords, :splits_text_on, :normalizes_words, :removes_characters_after_splitting, :to => :default_index
+    
+    delegate :removes_characters, :contracts_expressions, :stopwords, :splits_text_on, :normalizes_words, :removes_characters_after_splitting, :to => :default_tokenizer
     
     # Delegates.
     #

data/lib/picky/cores.rb

@@ -25,8 +25,7 @@ class Cores
     
     # 
     #
-    while generator
-      
+    loop do
       # Ramp it up to num processors.
       #
       while currently_processing < max
@@ -42,6 +41,7 @@ class Cores
         break unless element
         
         Process.fork do
+          sleep 0.01*currently_processing
           yield element
         end
         

data/lib/picky/extensions/array.rb

@@ -13,20 +13,10 @@ class Array
     end
   end
 
-  #
-  #
-  def map_with_index!
-    each_with_index do |element, index| self[index] = yield(element, index); end
-  end
-
-  #
-  #
-  def map_with_index &block
-    dup.map_with_index! &block
-  end
-
   # Accesses a random element of this array.
   #
+  # TODO Remove?
+  #
   def random
     self[Kernel.rand(self.length)]
   end

data/lib/picky/generator.rb

@@ -8,7 +8,29 @@ module Picky
   #   picky <command> <options>
   # is found.
   #
-  class NoGeneratorException < Exception; end
+  class NoGeneratorError < StandardError
+    
+    def initialize generator
+      super usage + possible_commands(generator.types)
+    end
+    
+    def usage
+      "\nUsage:\n" +
+      "picky <command> <params>\n" +
+      ?\n
+    end
+    
+    def possible_commands types
+      "Possible commands:\n" +
+      types.map do |name, klass_params|
+        result = "picky #{name}"
+        _, params = *klass_params
+        result << ' ' << [*params].map { |param| "<#{param}>" }.join(' ') if params
+        result
+      end.join(?\n) + ?\n
+    end
+    
+  end
   
   # This is a very simple project generator.
   # Not at all like Padrino's or Rails'.
@@ -22,7 +44,7 @@ module Picky
     
     def initialize
       @types = {
-        :project => Project
+        :project => [Project, :project_name]
       }
     end
     
@@ -38,8 +60,9 @@ module Picky
     #
     #
     def generator_for identifier, *args
-      generator_class = types[identifier.to_sym]
-      raise NoGeneratorException unless generator_class
+      generator_info = types[identifier.to_sym]
+      raise NoGeneratorError.new(self) unless generator_info
+      generator_class = generator_info.first
       generator_for_class generator_class, identifier, *args
     end
     

data/lib/picky/index/bundle.rb

@@ -8,11 +8,13 @@ module Index
   #
   class Bundle
     
+    attr_reader   :checker
     attr_reader   :name,             :category,         :type
     attr_accessor :index,            :weights,          :similarity
     attr_accessor :partial_strategy, :weights_strategy, :similarity_strategy
     
     delegate :[], :[]=, :clear, :to => :index
+    delegate :raise_unless_cache_exists, :to => :checker
     
     # Path is in which directory the cache is located.
     #
@@ -28,6 +30,8 @@ module Index
       @partial_strategy    = partial_strategy
       @weights_strategy    = weights_strategy
       @similarity_strategy = similarity_strategy
+      
+      @checker = BundleChecker.new self
     end
     
     # Get the ids for the text.
@@ -59,46 +63,6 @@ module Index
       File.join PICKY_ROOT, 'index'
       # category.search_index_root
     end
-    
-    def size_of path
-      `ls -l #{path} | awk '{print $5}'`.to_i
-    end
-    # Check if the cache files are there and do not have size 0.
-    #
-    def caches_ok?
-      cache_ok?(index_cache_path) &&
-      cache_ok?(similarity_cache_path) &&
-      cache_ok?(weights_cache_path)
-    end
-    # Is the cache ok? I.e. larger than four in size.
-    #
-    def cache_ok? path
-      size_of(path) > 0
-    end
-    # Raises an appropriate error message.
-    #
-    def raise_cache_missing what
-      raise "#{what} cache for #{identifier} missing."
-    end
-    # Is the cache small?
-    #
-    def cache_small? path
-      size_of(path) < 16
-    end
-    def warn_cache_small what
-      puts "#{what} cache for #{identifier} smaller than 16 bytes."
-    end
-    # Check all index files and raise if necessary.
-    #
-    def raise_unless_cache_exists
-      warn_cache_small :index      if cache_small?(index_cache_path)
-      # warn_cache_small :similarity if cache_small?(similarity_cache_path)
-      warn_cache_small :weights    if cache_small?(weights_cache_path)
-
-      raise_cache_missing :index      unless cache_ok?(index_cache_path)
-      raise_cache_missing :similarity unless cache_ok?(similarity_cache_path)
-      raise_cache_missing :weights    unless cache_ok?(weights_cache_path)
-    end
 
     # Copies the indexes to the "backup" directory.
     #
@@ -257,7 +221,7 @@ module Index
     # TODO Use config object?
     #
     def search_index_file_name
-      File.join cache_directory, "#{type.name}_#{category.name}_index.txt"
+      File.join cache_directory, "prepared_#{category.name}_index.txt"
     end
     
     # Generators.

data/lib/picky/index/bundle_checker.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+#
+module Index
+
+  # Checks bundle indexes.
+  #
+  class BundleChecker
+    
+    attr_reader :bundle
+    
+    def initialize bundle
+      @bundle = bundle
+    end
+    
+    # Check all index files and raise if necessary.
+    #
+    def raise_unless_cache_exists
+      warn_cache_small :index      if cache_small?(bundle.index_cache_path)
+      # warn_cache_small :similarity if cache_small?(similarity_cache_path)
+      warn_cache_small :weights    if cache_small?(bundle.weights_cache_path)
+
+      raise_cache_missing :index      unless cache_ok?(bundle.index_cache_path)
+      raise_cache_missing :similarity unless cache_ok?(bundle.similarity_cache_path)
+      raise_cache_missing :weights    unless cache_ok?(bundle.weights_cache_path)
+    end
+    
+    def size_of path
+      `ls -l #{path} | awk '{print $5}'`.to_i
+    end
+    # Check if the cache files are there and do not have size 0.
+    #
+    def caches_ok?
+      cache_ok?(bundle.index_cache_path) &&
+      cache_ok?(bundle.similarity_cache_path) &&
+      cache_ok?(bundle.weights_cache_path)
+    end
+    # Is the cache ok? I.e. larger than four in size.
+    #
+    def cache_ok? path
+      size_of(path) > 0
+    end
+    # Raises an appropriate error message.
+    #
+    def raise_cache_missing what
+      raise "#{what} cache for #{bundle.identifier} missing."
+    end
+    # Is the cache small?
+    #
+    def cache_small? path
+      size_of(path) < 16
+    end
+    def warn_cache_small what
+      puts "#{what} cache for #{bundle.identifier} smaller than 16 bytes."
+    end
+    
+  end
+  
+end

data/lib/picky/index/type.rb

@@ -16,7 +16,10 @@ module Index
       @name        = name
       @result_type = result_type # TODO Move.
       @categories  = categories # for each_delegate
-      @combinator  = Query::Combinator.new @categories, :ignore_unassigned_tokens => ignore_unassigned_tokens # TODO pass this in?
+      @combinator  = combinator_for categories, ignore_unassigned_tokens
+    end
+    def combinator_for categories, ignore_unassigned_tokens
+       Query::Combinator.new @categories, :ignore_unassigned_tokens => ignore_unassigned_tokens
     end
     
     #

data/lib/picky/index/wrappers/exact_first.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+#
+module Index
+  
+  module Wrappers
+    
+    # This index combines an exact and partial index.
+    # It serves to order the results such that exact  hits are found first.
+    #
+    # TODO Need to use the right subtokens. Bake in?
+    #
+    class ExactFirst < Bundle
+      
+      delegate :similar,
+               :identifier,
+               :name,
+               :to => :@exact
+      delegate :type,
+               :category,
+               :weight,
+               :generate_partial_from,
+               :generate_caches_from_memory,
+               :generate_derived,
+               :dump,
+               :load,
+               :to => :@partial
+      
+      def initialize category
+        @exact   = category.exact
+        @partial = category.partial
+      end
+      
+      def self.wrap type_or_category
+        if type_or_category.respond_to? :categories
+          wrap_each_of type_or_category.categories
+          type_or_category
+        else
+          new type_or_category
+        end
+      end
+      def self.wrap_each_of categories
+        categories.collect! { |category| new(category) }
+      end
+      
+      def ids text
+        @exact.ids(text) + @partial.ids(text)
+      end
+      
+      def weight text
+        [@exact.weight(text) || 0, @partial.weight(text) || 0].max
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/indexes.rb

@@ -11,16 +11,17 @@ module Indexes
   end
   # Runs the indexers in parallel (index + cache).
   #
-  def self.index
+  def self.index randomly = true
     Indexes.take_snapshot
 
     # Run in parallel.
     #
-    timed_exclaim "INDEXING USING #{Cores.max_processors} PROCESSORS."
-    Cores.forked self.fields, :randomly => true do |field|
-      # Reestablish DB connection.
-      #
-      connect_backends
+    # TODO Make option to also use non-random.
+    #      rake index:random (default)
+    #      rake index:ordered
+    #
+    timed_exclaim "INDEXING USING #{Cores.max_processors} PROCESSORS, IN #{randomly ? 'RANDOM' : 'GIVEN'} ORDER."
+    Cores.forked self.fields, { :randomly => randomly } do |field, cores|
       field.index
       field.cache
     end
@@ -30,14 +31,6 @@ module Indexes
     configuration.index_solr
   end
   
-  # TODO Push into configuration.
-  #
-  def self.connect_backends
-    configuration.types.each do |type|
-      type.connect_backend
-    end
-  end
-  
   # Returns an array of fields.
   #
   # TODO Rewrite.
@@ -111,6 +104,8 @@ module Indexes
   end
   # Loads all indexes from the caches.
   #
+  # TODO Rename load_indexes.
+  #
   def self.load_from_cache
     each &:load_from_cache
   end
@@ -132,11 +127,9 @@ module Indexes
   # Removes the cache files.
   #
   def self.clear_caches
-    each do |type|
-      type.categories.each do |category|
-        category.exact.delete_all
-        category.partial.delete_all
-      end
+    each_bundle do |exact, partial|
+      exact.delete_all
+      partial.delete_all
     end
   end
 

data/lib/picky/loader.rb

@@ -161,7 +161,9 @@ module Loader
     load_relative 'index/category'
     load_relative 'index/type'
     
-    load_relative 'index/combined'
+    load_relative 'index/bundle_checker'
+    
+    load_relative 'index/wrappers/exact_first'
     
     # Tokens.
     #
@@ -173,7 +175,9 @@ module Loader
     load_relative 'tokenizers/base'
     load_relative 'tokenizers/index'
     load_relative 'tokenizers/query'
-    load_relative 'tokenizers/default'
+    
+    load_relative 'tokenizers/default/index'
+    load_relative 'tokenizers/default/query'
     
     # Query combinations, qualifiers, weigher.
     #
@@ -208,6 +212,7 @@ module Loader
     load_relative 'sources/base'
     load_relative 'sources/db'
     load_relative 'sources/csv'
+    load_relative 'sources/delicious'
     
     # Indexes.
     #
@@ -237,11 +242,5 @@ module Loader
     #
     load_relative 'generator'
   end
-  
-  # Silenceable puts.
-  #
-  def self.exclaim text
-    puts text
-  end
 
 end

data/lib/picky/query/allocation.rb

@@ -61,7 +61,7 @@ module Query
     # Transform the allocation into result form.
     #
     def to_result
-      [self.result_type, self.score, count, @combinations.to_result, self.ids] if count = self.count > 0
+      [self.result_type, self.score, self.count, @combinations.to_result, self.ids] if count = self.count > 0
     end
 
     # Json representation of this allocation.

data/lib/picky/query/combinations.rb

@@ -19,14 +19,17 @@ module Query
       @combinations.hash
     end
 
-    #
-    #
-    # TODO Rewrite.
+    # Uses user specific weights to calculate a score for the combinations.
     #
     def calculate_score weights
-      @score  = @combinations.sum &:weight
-      @score += weights.score @combinations
-      @score
+      @score ||= sum_score
+      @score + add_score(weights) # TODO Ok to just cache the weights?
+    end
+    def sum_score
+      @combinations.sum &:weight
+    end
+    def add_score weights
+      weights.score @combinations
     end
 
     # Gets all ids for the allocations.

data/lib/picky/query/combinator.rb

@@ -31,13 +31,19 @@ module Query
       token.similar? ? similar_possible_for(token) : possible_for(token)
     end
     
-    # TODO Break apart.
-    #
+    # 
+    # 
     def similar_possible_for token
       # Get as many similar tokens as necessary
       #
+      tokens = similar_tokens_for token
+      # possible combinations
+      #
+      inject_possible_for tokens
+    end
+    def similar_tokens_for token
       text = token.text
-      tokens = categories.inject([]) do |result, category|
+      categories.inject([]) do |result, category|
         next_token = token
         # TODO adjust either this or the amount of similar in index
         #
@@ -46,8 +52,8 @@ module Query
         end
         result
       end
-      # possible combinations
-      #
+    end
+    def inject_possible_for tokens
       tokens.inject([]) do |result, token|
         possible = possible_categories token
         result + possible_for(token, possible)

data/lib/picky/rack/harakiri.rb

@@ -32,7 +32,7 @@ module Rack
     #
     def harakiri
       @requests = @requests + 1
-      Process.kill(:QUIT, Process.pid) if @requests > @quit_after_requests
+      Process.kill(:QUIT, Process.pid) if @requests >= @quit_after_requests
     end
     
   end

data/lib/picky/results/base.rb

@@ -17,21 +17,13 @@ module Results
       @allocations = allocations || Query::Allocations.new
     end
 
-    def add more_results
-      @added = more_results
-      self
-    end
-    def added
-      @added || {}
-    end
-
     #
     #
     def serialize
-      added.merge(:allocations => allocations.to_result,
-                  :offset      => offset,
-                  :duration    => duration,
-                  :total       => total)
+      { :allocations => allocations.to_result,
+        :offset      => offset,
+        :duration    => duration,
+        :total       => total }
     end
     # The default format is json.
     #

data/lib/picky/results/live.rb

@@ -9,11 +9,5 @@ module Results
       ?. + super
     end
     
-    # The default response style for live results is to_json.
-    #
-    def to_response
-      to_json
-    end
-    
   end
 end

data/lib/picky/routing.rb

@@ -32,23 +32,23 @@ class Routing
     routes.call env
   end
   
-  # Set the defaults.
-  # 
-  # Options are:
-  #  * :query_key  => :query # default
-  #  * :offset_key => :offset # default
-  #  
-  #  * :tokenizer  => Tokenizers::Query.new # default
-  #
-  def defaults options = {}
-    @defaults[:query_key]    = options[:query_key].to_s  if options[:query_key]
-    @defaults[:offset_key]   = options[:offset_key].to_s if options[:offset_key]
-    
-    @defaults[:tokenizer]    = options[:tokenizer]       if options[:tokenizer]
-    @defaults[:content_type] = options[:content_type]    if options[:content_type]
-    
-    @defaults
-  end
+  # # Set the defaults.
+  # # 
+  # # Options are:
+  # #  * :query_key  => :query # default
+  # #  * :offset_key => :offset # default
+  # #  
+  # #  * :tokenizer  => Tokenizers::Query.new # default
+  # #
+  # def defaults options = {}
+  #   @defaults[:query_key]    = options[:query_key].to_s  if options[:query_key]
+  #   @defaults[:offset_key]   = options[:offset_key].to_s if options[:offset_key]
+  #   
+  #   @defaults[:tokenizer]    = options[:tokenizer]       if options[:tokenizer]
+  #   @defaults[:content_type] = options[:content_type]    if options[:content_type]
+  #   
+  #   @defaults
+  # end
   
   #
   #