picky 1.5.2 → 1.5.3

data/lib/picky/analyzer.rb

@@ -0,0 +1,154 @@
+# encoding: utf-8
+#
+
+# Analyzes indexes (index bundles, actually).
+#
+# Can be output using to_s.
+#
+class Analyzer
+
+  attr_reader :analysis, :comments
+
+  #
+  #
+  def initialize
+    @analysis = {}
+    @comments = []
+  end
+
+  #
+  #
+  def analyze bundle
+    bundle.load_index
+    analysis[:__keys] = bundle.size
+    cardinality :index, bundle.index
+    index_analysis
+    bundle.clear_index
+
+    bundle.load_weights
+    weights bundle.weights
+    weights_analysis
+    bundle.clear_weights
+
+    bundle.load_similarity
+    cardinality :similarity, bundle.similarity
+    bundle.clear_similarity
+
+    # bundle.load_configuration
+    # analysis[:configuration] = bundle.configuration
+    # bundle.clear_configuration
+
+    self
+  end
+  def cardinality identifier, index
+    return if index.size.zero?
+
+    key_length_average = 0
+    ids_length_average = 0
+
+    min_ids_length = 1.0/0 # Infinity
+    max_ids_length =     0
+    min_key_length = 1.0/0 # Infinity
+    max_key_length =     0
+
+    key_size, ids_size =
+    index.each_pair do |key, ids|
+      key_size = key.size
+      if key_size < min_key_length
+        min_key_length = key_size
+      else
+        max_key_length = key_size if key_size > max_key_length
+      end
+      key_length_average += key_size
+
+      ids_size = ids.size
+      if ids_size < min_ids_length
+        min_ids_length = ids_size
+      else
+        max_ids_length = ids_size if ids_size > max_ids_length
+      end
+      ids_length_average += ids_size
+    end
+    index_size = index.size
+    key_length_average = key_length_average.to_f / index_size
+    ids_length_average = ids_length_average.to_f / index_size
+
+    analysis[identifier] ||= {}
+    analysis[identifier][:key_length]         = (min_key_length..max_key_length)
+    analysis[identifier][:ids_length]         = (min_ids_length..max_ids_length)
+    analysis[identifier][:key_length_average] = key_length_average
+    analysis[identifier][:ids_length_average] = ids_length_average
+  end
+  def index_analysis
+    return unless analysis[:index]
+
+    if analysis[:__keys] < 100
+      comments << "\033[33mVery small index (< 100 keys).\033[m"
+    end
+
+    range = analysis[:index][:key_length]
+    case range.min
+    when 1
+      comments << "\033[33mIndex matches single characters.\033[m"
+    end
+  end
+  def weights index
+    return if index.size.zero?
+
+    min_weight = 1.0/0 # Infinity
+    max_weight =   0.0
+
+    weight_average = 0
+
+    index.each_pair do |key, value|
+      if value < min_weight
+        min_weight = value
+      else
+        max_weight = value if value > max_weight
+      end
+      weight_average += value
+    end
+
+    weight_average = weight_average / index.size
+
+    analysis[:weights] ||= {}
+    analysis[:weights][:weight_range]   = (min_weight..max_weight)
+    analysis[:weights][:weight_average] = weight_average
+  end
+  def weights_analysis
+    return unless analysis[:weights]
+
+    range = analysis[:weights][:weight_range]
+
+    case range.max
+    when 0.0
+      comments << "\033[31mThere's only one id per key – you'll only get single results.\033[m"
+    end
+  end
+
+  #
+  #
+  def to_s
+    [*comments, index_to_s, weights_to_s, similarity_to_s, configuration_to_s].compact.join "\n"
+  end
+  def index_to_s
+    return if analysis[:__keys].zero?
+    [
+      "index key cardinality:                #{"%10d" % analysis[:__keys]}",
+      "index key length range (avg):         #{"%10s" % analysis[:index][:key_length]} (#{analysis[:index][:key_length_average].round(2)})",
+      "index ids per key length range (avg): #{"%10s" % analysis[:index][:ids_length]} (#{analysis[:index][:ids_length_average].round(2)})"
+    ].join("\n")
+  end
+  def weights_to_s
+    return unless analysis[:weights]
+    %Q{weights range (avg):                  #{"%10s" % analysis[:weights][:weight_range]} (#{analysis[:weights][:weight_average].round(2)})}
+  end
+  def similarity_to_s
+    return unless analysis[:similarity]
+    %Q{similarity key length range (avg):    #{"%10s" % analysis[:similarity][:key_length]} (#{analysis[:similarity][:key_length_average].round(2)})}
+  end
+  def configuration_to_s
+    # analysis[:configuration]
+  end
+
+end

data/lib/picky/application.rb

@@ -19,12 +19,12 @@
 #   my_index = Index::Memory.new :some_index_name, some_source
 # You give the index a name (or identifier), and a source (see Sources), where its data comes from. Let's do that:
 #   class MyGreatSearch < Application
-#     
+#
 #     books = Index::Memory.new :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
-#     
+#
 #   end
 # Now we have an index <tt>books</tt>.
-# 
+#
 # That on itself won't do much good.
 #
 # Note that a Redis index is also available: Index::Redis.new.
@@ -38,7 +38,7 @@
 #
 # Let's go ahead and define a category:
 #   class MyGreatSearch < Application
-#     
+#
 #     books = Index::Memory.new :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title
 #
@@ -56,16 +56,16 @@
 #    full_books_query = Query::Full.new books
 # Full just means that the ids are returned with the results.
 # Picky also offers a Query that returns live results, Query::Live. But that's not important right now.
-# 
+#
 # Now we have somebody we can ask about the index. But no external interface.
-# 
+#
 # == route(/regexp1/ => query1, /regexp2/ => query2, ...)
 #
 # Let's add a URL path (a Route, see http://github.com/floere/picky/wiki/Routing-configuration) to which we can send our queries. We do that with the route method:
 #   route %r{^/books/full$} => full_books_query
 # In full glory:
 #   class MyGreatSearch < Application
-#     
+#
 #     books = index :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title
 #
@@ -110,7 +110,7 @@
 #
 # Our example, fully fleshed out with indexing, querying, and weights:
 #   class MyGreatSearch < Application
-#     
+#
 #     default_indexing removes_characters: /[^a-zA-Z0-9\.]/,
 #                      stopwords: /\b(and|or|in|on|is|has)\b/,
 #                      splits_text_on: /\s/,
@@ -127,7 +127,7 @@
 #                      removes_characters_after_splitting: /\./,
 #                      substitutes_characters_with: CharacterSubstituters::WestEuropean.new,
 #                      maximum_tokens: 4
-#     
+#
 #     books = Index::Memory.new :books, Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #     books.define_category :title,
 #                           qualifiers: [:t, :title, :titre],
@@ -136,36 +136,36 @@
 #     books.define_category :author,
 #                           partial: Partial::Substring.new(:from => -2)
 #     books.define_category :isbn
-#     
+#
 #     query_options = { :weights => { [:title, :author] => +3, [:author, :title] => -1 } }
-#     
+#
 #     route %r{^/books/full$} => Query::Full.new(books, query_options)
 #     route %r{^/books/live$} => Query::Live.new(books, query_options)
-#     
+#
 #   end
 # That's actually already a full-blown Picky App!
 #
 class Application
-  
+
   class << self
-    
+
     # API
     #
-    
+
     # Returns a configured tokenizer that
     # is used for indexing by default.
-    # 
+    #
     def default_indexing options = {}
       Internals::Tokenizers::Index.default = Internals::Tokenizers::Index.new(options)
     end
-    
+
     # Returns a configured tokenizer that
     # is used for querying by default.
-    # 
+    #
     def default_querying options = {}
       Internals::Tokenizers::Query.default = Internals::Tokenizers::Query.new(options)
     end
-    
+
     # Create a new index for indexing and for querying.
     #
     # Parameters:
@@ -176,22 +176,23 @@ class Application
     # * source: The source the data comes from. See Sources::Base.
     #
     # Options:
-    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index. 
+    # * result_identifier: Use if you'd like a different identifier/name in the results JSON than the name of the index.
     #
-    # TODO Obsolete. Phase out.
+    # TODO Remove in 1.6.
     #
     def index name, source, options = {}
+      raise "the Picky application method #index is deprecated, please use Index::Memory.new instead."
       Index::Memory.new name, source, options
     end
-    
+
     # Routes.
     #
     delegate :route, :root, :to => :rack_adapter
-    
+
     #
     # API
-    
-    
+
+
     # A Picky application implements the Rack interface.
     #
     # Delegates to its routing to handle a request.
@@ -202,7 +203,7 @@ class Application
     def rack_adapter # :nodoc:
       @rack_adapter ||= Internals::FrontendAdapters::Rack.new
     end
-    
+
     # Finalize the subclass as soon as it
     # has finished loading.
     #
@@ -233,18 +234,37 @@ class Application
     def check # :nodoc:
       warnings = []
       warnings << check_external_interface
-      puts "\n#{warnings.join(?\n)}\n\n" unless warnings.all? &:nil?
+      warn "\n#{warnings.join(?\n)}\n\n" unless warnings.all? &:nil?
     end
     def check_external_interface
       "WARNING: No routes defined for application configuration in #{self.class}." if rack_adapter.empty?
     end
-    
-    # TODO Add more info if possible.
-    #
+
     def to_s # :nodoc:
-      "#{self.name}:\n#{rack_adapter}"
+      <<-APPLICATION
+\033[1m#{name}\033[m
+#{to_stats.indented_to_s}
+APPLICATION
+    end
+    def to_stats
+      <<-APP
+\033[1mIndexing (default)\033[m:
+#{Internals::Tokenizers::Index.default.indented_to_s}
+
+\033[1mQuerying (default)\033[m:
+#{Internals::Tokenizers::Query.default.indented_to_s}
+
+\033[1mIndexes\033[m:
+#{Indexes.to_s.indented_to_s}
+
+\033[1mRoutes\033[m:
+#{to_routes.indented_to_s}
+APP
+    end
+    def to_routes
+      rack_adapter.to_s
     end
-    
+
   end
-  
+
 end

data/lib/picky/character_substituters/west_european.rb

@@ -6,28 +6,32 @@ module CharacterSubstituters # :nodoc:all
   # (and more, see specs)
   #
   class WestEuropean
-    
+
     def initialize
       @chars = ActiveSupport::Multibyte.proxy_class
     end
-    
+
+    def to_s
+      self.class.name
+    end
+
     def substitute text
       trans = @chars.new(text).normalize(:kd)
-      
+
       # substitute special cases
       #
       trans.gsub!('ß', 'ss')
-      
+
       # substitute umlauts (of A,O,U,a,o,u)
       #
       trans.gsub!(/([AOUaou])\314\210/u, '\1e')
-      
+
       # get rid of ecutes, graves and …
       #
       trans.unpack('U*').select { |cp|
         cp < 0x0300 || cp > 0x035F
       }.pack('U*')
     end
-    
+
   end
 end

data/lib/picky/cli.rb

@@ -1,9 +1,9 @@
 module Picky
-  
+
   # A very simple CLI selector.
   #
   class CLI # :nodoc:all
-    
+
     # Execute a command.
     #
     # Note: By default, help is displayed. I.e. when no command is given.
@@ -16,7 +16,7 @@ module Picky
     def executor_class_for selector = nil
       selector && @@mapping[selector.to_sym] || [Help]
     end
-    
+
     class Base
       def usage name, params
         puts "Usage\n  picky #{name} #{params_to_s(params)}"
@@ -31,20 +31,20 @@ module Picky
       def execute name, args, params
         relative_log_file = args.shift
         port              = args.shift
-        
+
         usage(name, params) || exit(1) unless relative_log_file
-        
-        ENV['PICKY_LOG_FILE']        = File.expand_path relative_log_file  
+
+        ENV['PICKY_LOG_FILE']        = File.expand_path relative_log_file
         ENV['PICKY_STATISTICS_PORT'] = port
-        
+
         begin
           require 'picky-statistics'
         rescue LoadError => e
           require 'picky/extensions/object'
-          puts_gem_missing 'picky-statistics', 'the Picky statistics'
+          warn_gem_missing 'picky-statistics', 'the Picky statistics'
           exit 1
         end
-        
+
         require 'picky-statistics/application/app'
       end
     end
@@ -52,20 +52,20 @@ module Picky
       def execute name, args, params
         url  = args.shift
         port = args.shift
-        
+
         usage(name, params) || exit(1) unless args.empty?
-        
+
         ENV['PICKY_LIVE_URL']  = url
         ENV['PICKY_LIVE_PORT'] = port
-        
+
         begin
           require 'picky-live'
         rescue LoadError => e
           require 'picky/extensions/object'
-          puts_gem_missing 'picky-live', 'the Picky Live Interface'
+          warn_gem_missing 'picky-live', 'the Picky Live Interface'
           exit 1
         end
-        
+
         require 'picky-live/application/app'
       end
     end
@@ -82,11 +82,11 @@ module Picky
           _, *params = object_and_params
           "  picky #{command} #{params_to_s(params)}"
         end.join(?\n)
-        
+
         puts "Possible commands:\n" + commands
       end
     end
-    
+
     # Maps commands to the other gem's command.
     #
     @@mapping = {
@@ -98,7 +98,7 @@ module Picky
     def self.mapping
       @@mapping
     end
-    
+
   end
-  
+
 end

data/lib/picky/index/base.rb

@@ -1,12 +1,12 @@
 module Index
-  
+
   # This class defines the indexing and index API that is exposed to the user
   # as the #index method inside the Application class.
   #
   # It provides a single front for both indexing and index options. We suggest to always use the index API.
   #
   # Note: An Index holds both an *Indexed*::*Index* and an *Indexing*::*Type*.
-  #  
+  #
   class Base
 
     attr_reader :name, :indexing, :indexed
@@ -22,14 +22,45 @@ module Index
     # * after_indexing: As of this writing only used in the db source. Executes the given after_indexing as SQL after the indexing process.
     #
     def initialize name, source, options = {}
-      @name     = name
+      check name, source
+
+      @name     = name.to_sym
       @indexing = Internals::Indexing::Index.new name, source, options
       @indexed  = Internals::Indexed::Index.new  name, options
-  
+
       # Centralized registry.
       #
       Indexes.register self
     end
+    #
+    # Since this is an API, we fail hard quickly.
+    #
+    def check name, source
+      raise ArgumentError.new(<<-NAME
+The index identifier (you gave "#{name}") for Index::Memory/Index::Redis should be a String/Symbol,
+Examples:
+  Index::Memory.new(:my_cool_index, ...) # Recommended
+  Index::Redis.new("a-redis-index", ...)
+NAME
+) unless name.respond_to?(:to_sym)
+      raise ArgumentError.new(<<-SOURCE
+The index "#{name}" should use a data source that responds to the method #harvest, which yields(id, text).
+Or it could use one of the built-in sources:
+  Sources::#{(Sources.constants - [:Base, :Wrappers, :NoCSVFileGiven, :NoCouchDBGiven]).join(',
+  Sources::')}
+SOURCE
+) unless source.respond_to?(:harvest)
+    end
+
+    def to_stats
+      stats = <<-INDEX
+#{name} (#{self.class}):
+  #{"source:            #{indexing.source}".indented_to_s}
+  #{"categories:        #{indexing.categories.categories.map(&:name).join(', ')}".indented_to_s}
+INDEX
+      stats << "  result identifier: \"#{indexed.result_identifier}\"".indented_to_s unless indexed.result_identifier.to_s == indexed.name.to_s
+      stats
+    end
 
     # Defines a searchable category on the index.
     #
@@ -46,12 +77,12 @@ module Index
     #
     def define_category category_name, options = {}
       category_name = category_name.to_sym
-  
+
       indexing_category = indexing.define_category category_name, options
       indexed_category  = indexed.define_category  category_name, options
-  
+
       yield indexing_category, indexed_category if block_given?
-  
+
       self
     end
     alias category define_category
@@ -87,7 +118,7 @@ module Index
     #   x:133, y:120
     #
     # This will search this square area (* = 133, 120: The "search" point entered):
-    # 
+    #
     #    132       134
     #     |         |
     #   --|---------|-- 121
@@ -95,7 +126,7 @@ module Index
     #     |    *    |
     #     |         |
     #   --|---------|-- 119
-    #     |         | 
+    #     |         |
     #
     # Note: The area does not need to be square, but can be rectangular.
     #
@@ -117,13 +148,13 @@ module Index
     #
     def define_ranged_category category_name, range, options = {}
       precision = options[:precision]
-  
+
       options = { partial: Partial::None.new }.merge options
-  
+
       define_category category_name, options do |indexing, indexed|
         indexing.source    = Sources::Wrappers::Location.new indexing, grid: range, precision: precision
         indexing.tokenizer = Internals::Tokenizers::Index.new
-    
+
         exact_bundle    = Indexed::Wrappers::Bundle::Location.new indexed.exact, grid: range, precision: precision
         indexed.exact   = exact_bundle
         indexed.partial = exact_bundle # A partial token also uses the exact index.
@@ -175,5 +206,5 @@ module Index
     end
     alias map_location define_map_location
   end
-  
+
 end

data/lib/picky/index_bundle.rb

@@ -4,7 +4,12 @@ class IndexBundle # :nodoc:all
 
   attr_reader :indexes, :index_mapping, :indexing, :indexed
 
-  delegate :reload,
+  delegate :size,
+           :each,
+           :to => :indexes
+
+  delegate :analyze,
+           :reload,
            :load_from_cache,
            :to => :indexed
 
@@ -19,22 +24,26 @@ class IndexBundle # :nodoc:all
   def initialize
     @indexes = []
     @index_mapping = {}
-  
+
     @indexed  = Indexed::Indexes.new
     @indexing = Indexing::Indexes.new
   end
 
+  def to_s
+    indexes.map &:to_stats
+  end
+
   def register index
     self.indexes << index
     self.index_mapping[index.name] = index
-  
+
     indexing.register index.indexing
     indexed.register  index.indexed
   end
 
   def [] name
     name = name.to_sym
-  
+
     self.index_mapping[name]
   end