picky 1.5.2 → 1.5.3

data/lib/picky/query/base.rb

@@ -20,12 +20,13 @@ module Query
   # Not directly instantiated. However, its methods are used by its subclasses, Full and Live.
   #
   class Base
-  
+
     include Helpers::Measuring
-  
+
+    attr_reader   :indexes
     attr_writer   :tokenizer, :identifiers_to_remove
     attr_accessor :reduce_to_amount, :weights
-  
+
     # Takes:
     # * A number of indexes
     # * Options hash (optional) with:
@@ -40,7 +41,7 @@ module Query
       weights      = options[:weights] || Weights.new
       @weights     = Hash === weights ? Weights.new(weights) : weights
     end
-    
+
     # Returns the right combinations strategy for
     # a number of query indexes.
     #
@@ -73,7 +74,7 @@ module Query
     def raise_different index_types
       raise DifferentTypesError.new(index_types)
     end
-  
+
     # This is the main entry point for a query.
     # Use this in specs and also for running queries.
     #
@@ -86,22 +87,22 @@ module Query
     def search_with_text text, offset = 0
       search tokenized(text), offset
     end
-  
+
     # Runs the actual search using Query::Tokens.
     #
     # Note: Internal method, use #search_with_text.
     #
     def search tokens, offset = 0
       results = nil
-    
+
       duration = timed do
-        results = execute(tokens, offset) || empty_results(offset) # TODO Does not work yet
+        results = execute tokens, offset
       end
       results.duration = duration.round 6
-    
+
       results
     end
-  
+
     # Execute a search using Query::Tokens.
     #
     # Note: Internal method, use #search_with_text.
@@ -109,16 +110,7 @@ module Query
     def execute tokens, offset
       result_type.from offset, sorted_allocations(tokens)
     end
-  
-    # Returns an empty result with default values.
-    #
-    # Parameters:
-    # * offset = 0: _optional_ The offset to use for the empty results.
-    #
-    def empty_results offset = 0
-      result_type.new offset
-    end
-  
+
     # Delegates the tokenizing to the query tokenizer.
     #
     # Parameters:
@@ -127,7 +119,7 @@ module Query
     def tokenized text
       @tokenizer.tokenize text
     end
-  
+
     # Gets sorted allocations for the tokens.
     #
     # This generates the possible allocations, sorted.
@@ -144,27 +136,27 @@ module Query
       # TODO uniq, score, sort in there
       #
       allocations = @indexes.allocations_for tokens
-    
+
       # Callbacks.
       #
       # TODO Reduce before sort?
       #
       reduce allocations
       remove_from allocations
-    
+
       # Remove double allocations.
       #
       allocations.uniq
-    
+
       # Score the allocations using weights as bias.
       #
       allocations.calculate_score weights
-    
+
       # Sort the allocations.
       # (allocations are sorted according to score, highest to lowest)
       #
       allocations.sort
-    
+
       # Return the allocations.
       #
       allocations
@@ -172,7 +164,7 @@ module Query
     def reduce allocations # :nodoc:
       allocations.reduce_to reduce_to_amount if reduce_to_amount
     end
-  
+
     #
     #
     def remove_from allocations # :nodoc:
@@ -183,12 +175,14 @@ module Query
     def identifiers_to_remove # :nodoc:
       @identifiers_to_remove ||= []
     end
-  
+
     # Display some nice information for the user.
     #
     def to_s
-      s = "#{self.class}"
+      s = "#{self.class}("
+      s << @indexes.indexes.map(&:name).join(', ')
       s << ", weights: #{@weights}" unless @weights.empty?
+      s << ")"
       s
     end
 

data/lib/picky/sources/base.rb

@@ -7,12 +7,12 @@
 # * Delicious (http://del.icio.us, online bookmarking service)
 # See also:
 # http://github.com/floere/picky/wiki/Sources-Configuration
-# 
+#
 # Don't worry if your source isn't here. Adding your own is easy:
 # http://github.com/floere/picky/wiki/Contributing-sources
 #
 module Sources
-  
+
   # Sources are where your data comes from.
   #
   # A source has 1 mandatory and 2 optional methods:
@@ -24,9 +24,9 @@ module Sources
   # Subclass this class <tt>class MySource < Base</tt> and override the methods in your source to do something.
   #
   class Base
-    
+
     attr_reader :key_format
-    
+
     # Connect to the backend.
     #
     # Called once per index/category combination before harvesting.
@@ -37,9 +37,9 @@ module Sources
     # * We open an file with data.
     #
     def connect_backend
-      
+
     end
-    
+
     # Called by the indexer when gathering data.
     #
     # Yields the data (id, text for id) for the given type and category.
@@ -51,7 +51,7 @@ module Sources
     def harvest index, category # :yields: id, text_for_id
       # This concrete implementation yields "nothing", override in subclasses.
     end
-    
+
     # Used to take a snapshot of your data if it is fast changing.
     #
     # Called once for each type before harvesting.
@@ -60,9 +60,9 @@ module Sources
     # * In a DB source, a table based on the source's select statement is created.
     #
     def take_snapshot index
-      
+
     end
-    
+
   end
-  
+
 end

data/lib/picky/sources/couch.rb

@@ -1,12 +1,12 @@
 module Sources
-  
+
   # Raised when a Couch source is instantiated without a file.
   #
   # Example:
   #   Sources::Couch.new(:column1, :column2) # without file option
   #
   class NoCouchDBGiven < StandardError; end
-  
+
   # A Couch database source.
   #
   # Options:
@@ -19,28 +19,32 @@ module Sources
   #  Sources::Couch.new(:title, :author, :isbn, url:'localhost:5984', user:'someuser', password:'somepassword')
   #
   class Couch < Base
-    
+
     #
     #
     def initialize *category_names, options
       check_gem
-      
+
       Hash === options && options[:url] || raise_no_db_given(category_names)
       @db = RestClient::Resource.new options.delete(:url), options
     end
-    
-    # Default key format method for couch DB is to_sym. 
+
+    def to_s
+      self.class.name
+    end
+
+    # Default key format method for couch DB is to_sym.
     #
     def key_format
       :to_sym
     end
-    
+
     # Tries to require the rest_client gem.
     #
     def check_gem # :nodoc:
       require 'rest_client'
     rescue LoadError
-      puts_gem_missing 'rest-client', 'the CouchDB source'
+      warn_gem_missing 'rest-client', 'the CouchDB source'
       exit 1
     end
 
@@ -55,14 +59,14 @@ module Sources
         yield doc[@@id_key], doc[category_name] || next
       end
     end
-    
+
     def get_data &block # :nodoc:
       resp = @db['_all_docs?include_docs=true'].get
       JSON.parse(resp)['rows'].
         map{|row| row['doc']}.
         each &block
     end
-    
+
     def raise_no_db_given category_names # :nodoc:
       raise NoCouchDBGiven.new(category_names.join(', '))
     end

data/lib/picky/sources/csv.rb

@@ -1,14 +1,14 @@
 module Sources
-  
+
   # Raised when a CSV source is instantiated without a file.
   #
   # Example:
   #   Sources::CSV.new(:column1, :column2) # without file option
   #
   class NoCSVFileGiven < StandardError; end
-  
+
   # Describes a CSV source, a file with comma separated values in it.
-  # 
+  #
   # The first column is implicitly assumed to be the id column.
   #
   # It takes the same options as the Ruby 1.9 CSV class.
@@ -19,36 +19,43 @@ module Sources
   #  Sources::CSV.new(:title, :author, :isbn, file:'data/a_csv_file.csv', row_sep:"\n")
   #
   class CSV < Base
-    
+
     # The CSV file's path, relative to PICKY_ROOT.
     #
     attr_reader :file_name
-    
+
     # The options that were passed into #new.
     #
     attr_reader :csv_options, :key_format
-    
+
     # The data category names.
     #
     attr_reader :category_names
-    
+
     def initialize *category_names, options
       require 'csv'
       @category_names = category_names
-      
+
       @csv_options = Hash === options && options || {}
       @file_name   = @csv_options.delete(:file) || raise_no_file_given(category_names)
-      
+
       key_format   = options.delete :key_format
       @key_format  = key_format && key_format.to_sym || :to_i
     end
-    
+
+    def to_s
+      parameters = category_names
+      parameters << { file: file_name }
+      parameters << csv_options unless csv_options.empty?
+      %Q{#{self.class.name}(#{parameters.join(', ')})}
+    end
+
     # Raises a NoCSVFileGiven exception.
     #
     def raise_no_file_given category_names # :nodoc:
       raise NoCSVFileGiven.new(category_names.join(', '))
     end
-    
+
     # Harvests the data to index.
     #
     def harvest _, category
@@ -61,13 +68,13 @@ module Sources
         yield indexed_id, text
       end
     end
-    
+
     #
     #
     def get_data &block # :nodoc:
       ::CSV.foreach file_name, csv_options, &block
     end
-    
+
   end
-  
+
 end

data/lib/picky/sources/db.rb

@@ -1,5 +1,5 @@
 module Sources
-  
+
   # Describes a database source. Needs a SELECT statement
   # (with id in it), and a file option or the options from an AR config file.
   #
@@ -15,27 +15,33 @@ module Sources
   #  Sources::DB.new('SELECT id, title, author, year FROM books', adapter: 'mysql', host:'localhost', ...)
   #
   class DB < Base
-    
+
     # The select statement that was passed in.
     #
     attr_reader :select_statement
-    
+
     # The database adapter.
     #
     attr_reader :database
-    
+
     # The database connection options that were either passed in or loaded from the given file.
     #
-    attr_reader :connection_options
-    
+    attr_reader :connection_options, :options
+
     @@traversal_id = :__picky_id
-    
+
     def initialize select_statement, options = { file: 'app/db.yml' }
       @select_statement = select_statement
       @database         = create_database_adapter
       @options          = options
     end
-    
+
+    def to_s
+      parameters = [select_statement.inspect]
+      parameters << options unless options.empty?
+      %Q{#{self.class.name}(#{parameters.join(', ')})}
+    end
+
     # Creates a database adapter for use with this source.
     def create_database_adapter # :nodoc:
       # TODO Do not use ActiveRecord directly.
@@ -46,7 +52,7 @@ module Sources
       adapter_class.abstract_class = true
       adapter_class
     end
-    
+
     # Configure the backend.
     #
     # Options:
@@ -63,7 +69,7 @@ module Sources
       end
       self
     end
-    
+
     # Connect the backend.
     #
     # Will raise unless connection options have been given.
@@ -73,64 +79,64 @@ module Sources
       raise "Database backend not configured" unless connection_options
       database.establish_connection connection_options
     end
-    
+
     # Take a snapshot of the data.
     #
     # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
     #
     def take_snapshot index
       connect_backend
-      
+
       origin = snapshot_table_name index
       on_database = database.connection
-      
+
       # Drop the table if it exists.
       #
       on_database.drop_table origin if on_database.table_exists?(origin)
-      
+
       # The adapters currently do not support this.
       #
       on_database.execute "CREATE TABLE #{origin} AS #{select_statement}"
-      
+
       # Add a column that Picky uses to traverse the table's entries.
       #
       on_database.add_column origin, @@traversal_id, :primary_key, :null => :false
-      
+
       # Execute any special queries this index needs executed.
       #
       on_database.execute index.after_indexing if index.after_indexing
     end
-    
+
     # Counts all the entries that are used for the index.
     #
     def count index
       connect_backend
-      
+
       database.connection.select_value("SELECT COUNT(#{@@traversal_id}) FROM #{snapshot_table_name(index)}").to_i
     end
-    
+
     # The name of the snapshot table created by Picky.
     #
     def snapshot_table_name index
       "picky_#{index.name}_index"
     end
-    
+
     # Harvests the data to index in chunks.
     #
     def harvest index, category, &block
       connect_backend
-      
+
       (0..count(index)).step(chunksize) do |offset|
         get_data index, category, offset, &block
       end
     end
-    
+
     # Gets the data from the backend.
     #
     def get_data index, category, offset, &block # :nodoc:
-      
+
       select_statement = harvest_statement_with_offset index, category, offset
-      
+
       # TODO Rewrite ASAP.
       #
       if database.connection.adapter_name == "PostgreSQL"
@@ -146,29 +152,29 @@ module Sources
         end
       end
     end
-    
+
     # Builds a harvest statement for getting data to index.
     #
     def harvest_statement_with_offset index, category, offset
       statement = harvest_statement index, category
-      
+
       statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
-      
+
       "#{statement} st.#{@@traversal_id} > #{offset} LIMIT #{chunksize}"
     end
-    
+
     # The harvest statement used to pull data from the snapshot table.
     #
     def harvest_statement index, category
       "SELECT id, #{category.from} FROM #{snapshot_table_name(index)} st"
     end
-    
+
     # The amount of records that are loaded each chunk.
     #
     def chunksize
       25_000
     end
-    
+
   end
-  
+
 end

data/lib/picky/sources/delicious.rb

@@ -1,5 +1,5 @@
 module Sources
-  
+
   # Describes a Delicious (http://deli.cio.us) source.
   #
   # This source has a fixed set of categories:
@@ -11,7 +11,7 @@ module Sources
   #  Sources::CSV.new('usrnam', 'paswrd')
   #
   class Delicious < Base
-    
+
     def initialize username, password
       check_gem
       @username = username
@@ -20,21 +20,24 @@ module Sources
     def check_gem # :nodoc:
       require 'www/delicious'
     rescue LoadError
-      puts_gem_missing 'www-delicious', 'the delicious source'
+      warn_gem_missing 'www-delicious', 'the delicious source'
       exit 1
     end
-    
+
+    def to_s
+      self.class.name
+    end
+
     # Harvests the data to index.
     #
     def harvest _, category
       get_data do |indexed_id, data|
         text = data[category.from]
         next unless text
-        text.force_encoding 'utf-8' # TODO Still needed?
         yield indexed_id, text
       end
     end
-    
+
     #
     #
     def get_data # :nodoc:
@@ -50,7 +53,7 @@ module Sources
         yield @generated_id, data
       end
     end
-    
+
   end
-  
+
 end

data/lib/picky/sources/wrappers/base.rb

@@ -1,6 +1,8 @@
 module Sources
   
-  # TODO Document.
+  # Source wrappers can be used to rewrite data before it goes into the index.
+  #
+  # For example if you want to normalize data.
   #
   module Wrappers # :nodoc:all
     

data/lib/picky/statistics.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+#
+
+# Gathers different statistics
+# when methods are called.
+#
+# Can be output using to_s.
+#
+class Statistics
+
+  def self.instance
+    @statistics ||= new
+  end
+
+  def preamble
+    loc = lines_of_code File.open('app/application.rb').read
+
+    @preamble ||= <<-PREAMBLE
+\033[1mApplication(s)\033[m
+  Definition LOC:  #{"%4d" % loc}
+  Indexes defined: #{"%4d" % Indexes.size}
+PREAMBLE
+  end
+
+  # Gathers information about the application.
+  #
+  def application
+    preamble
+    @application = Application.apps.map &:indented_to_s
+  end
+
+  # Gathers information about the indexes.
+  #
+  def analyze
+    preamble
+
+    @indexes = ["\033[1mIndexes analysis\033[m:"]
+    Indexes.analyze.each_pair do |name, index|
+      @indexes << <<-ANALYSIS
+#{"#{name}:".indented_to_s}:
+#{"exact:\n#{index[:exact].indented_to_s}".indented_to_s(4)}
+#{"partial*:\n#{index[:partial].indented_to_s}".indented_to_s(4)}
+ANALYSIS
+    end
+    @indexes = @indexes.join "\n"
+  end
+
+  # Outputs all gathered statistics.
+  #
+  def to_s
+    <<-STATS
+
+Picky Configuration:
+
+#{[@preamble, @application, @indexes].compact.join("\n")}
+STATS
+  end
+
+  # Internal methods.
+  #
+
+  def lines_of_code text
+    text.scan(/^\s*[^#\s].*$/).size
+  end
+
+end

data/lib/tasks/application.rake

@@ -1,4 +1,7 @@
 # desc "Loads the application, including its configuration."
+#
+# Note: This is used by tasks to load the application (and the framework) as a dependency.
+#
 task :application => :framework do
   Loader.load_application
 end

data/lib/tasks/checks.rake

@@ -0,0 +1,11 @@
+# Checks to help the user.
+#
+namespace :check do
+
+  desc 'Checks the index files for files that are suspiciously small or missing.'
+  task :index => :application do
+    Indexes.check_caches
+    puts "All indexes look ok."
+  end
+
+end

data/lib/tasks/framework.rake

@@ -1,4 +1,7 @@
 # desc "Loads the framework."
+#
+# Note: This is used by tasks to load the framework as a dependency.
+#
 task :framework do
   require File.expand_path '../../picky', __FILE__
 end