picky 2.6.0 → 2.7.0

data/lib/picky/indexes_indexing.rb

@@ -2,30 +2,23 @@
 #
 class Indexes
 
-  instance_delegate :take_snapshot,
-                    :generate_caches,
-                    :backup_caches,
-                    :restore_caches,
-                    :check_caches,
-                    :clear_caches,
-                    :create_directory_structure,
-                    :index,
-                    :index_for_tests
-
-  each_delegate :take_snapshot,
-                :generate_caches,
-                :backup_caches,
-                :restore_caches,
-                :check_caches,
-                :clear_caches,
-                :create_directory_structure,
+  instance_delegate :index,
+                    :check,
+                    :clear,
+                    :backup,
+                    :restore,
+                    :index_for_tests,
+                    :tokenizer
+
+  each_delegate :check,
+                :clear,
+                :backup,
+                :restore,
                 :to => :indexes
 
   # Runs the indexers in parallel (prepare + cache).
   #
   def index randomly = true
-    take_snapshot
-
     # Run in parallel.
     #
     timed_exclaim "Indexing using #{Cores.max_processors} processors, in #{randomly ? 'random' : 'given'} order."
@@ -41,9 +34,13 @@ class Indexes
   # without forking and shouting ;)
   #
   def index_for_tests
-    take_snapshot
-
     indexes.each(&:index)
   end
 
+  # TODO Doc. Spec. Split into Query/Index.
+  #
+  def tokenizer
+    Tokenizers::Index.default
+  end
+
 end

data/lib/picky/indexing/bundle/base.rb

@@ -1,30 +1,48 @@
-# encoding: utf-8
-#
 module Indexing # :nodoc:all
 
+  # A Bundle is a number of indexes
+  # per [index, category] combination.
+  #
+  # At most, there are three indexes:
+  # * *core* index (always used)
+  # * *weights* index (always used)
+  # * *similarity* index (used with similarity)
+  #
+  # In Picky, indexing is separated from the index
+  # handling itself through a parallel structure.
+  #
+  # Both use methods provided by this base class, but
+  # have very different goals:
+  #
+  # * *Indexing*::*Bundle* is just concerned with creating index files
+  #   and providing helper functions to e.g. check the indexes.
+  #
+  # * *Index*::*Bundle* is concerned with loading these index files into
+  #   memory and looking up search data as fast as possible.
+  #
   module Bundle
 
     # This is the indexing bundle.
+    #
     # It does all menial tasks that have nothing to do
     # with the actual index running etc.
     #
-    class Base < SuperBase
+    class Base < ::Bundle
 
-      attr_accessor :partial_strategy, :weights_strategy
+      attr_accessor :partial_strategy,
+                    :weights_strategy
 
-      # Path is in which directory the cache is located.
-      #
-      def initialize name, category, similarity_strategy, partial_strategy, weights_strategy
+      def initialize name, category, weights_strategy, partial_strategy, similarity_strategy
         super name, category, similarity_strategy
 
-        @partial_strategy    = partial_strategy
-        @weights_strategy    = weights_strategy
+        @weights_strategy = weights_strategy
+        @partial_strategy = partial_strategy
       end
 
       # Sets up a piece of the index for the given token.
       #
-      def initialize_index_for token
-        index[token] ||= []
+      def initialize_inverted_index_for token
+        self.inverted[token] ||= []
       end
 
       # Generation
@@ -36,7 +54,7 @@ module Indexing # :nodoc:all
       # * Dumps all the indexes into files.
       #
       def generate_caches_from_source
-        load_from_index_file
+        load_from_prepared_index_file
         generate_caches_from_memory
       end
       # Generates derived indexes from the index and dumps.
@@ -60,13 +78,13 @@ module Indexing # :nodoc:all
 
       # Load the data from the db.
       #
-      def load_from_index_file
-        load_from_index_generation_message
+      def load_from_prepared_index_file
+        load_from_prepared_index_generation_message
         clear
         retrieve
       end
-      def load_from_index_generation_message
-        timed_exclaim %Q{"#{identifier}": Loading index.}
+      def load_from_prepared_index_generation_message
+        timed_exclaim %Q{"#{identifier}": Loading prepared data into memory.}
       end
       # Retrieves the prepared index data into the index.
       #
@@ -77,8 +95,8 @@ module Indexing # :nodoc:all
       def retrieve
         key_format = self[:key_format] || :to_i
         files.retrieve do |id, token|
-          initialize_index_for token
-          index[token] << id.send(key_format)
+          initialize_inverted_index_for token
+          self.inverted[token] << id.send(key_format)
         end
       end
 
@@ -86,63 +104,64 @@ module Indexing # :nodoc:all
       # partial caching strategy of this bundle.
       #
       def generate_partial
-        generator = Generators::PartialGenerator.new self.index
-        self.index = generator.generate self.partial_strategy
+        generator = Generators::PartialGenerator.new self.inverted
+        self.inverted = generator.generate self.partial_strategy
       end
-      # Generate a partial index from the given exact index.
+      # Generate a partial index from the given exact inverted index.
       #
-      def generate_partial_from exact_index
+      def generate_partial_from exact_inverted_index
         timed_exclaim %Q{"#{identifier}": Generating partial index for index.}
-        self.index = exact_index
+        self.inverted = exact_inverted_index
         self.generate_partial
         self
       end
-      # Generates a new similarity index (writes its index) using the
-      # given similarity caching strategy.
-      #
-      def generate_similarity
-        generator = Generators::SimilarityGenerator.new self.index
-        self.similarity = generator.generate self.similarity_strategy
-      end
       # Generates a new weights index (writes its index) using the
       # given weight caching strategy.
       #
       def generate_weights
-        generator = Generators::WeightsGenerator.new self.index
+        generator = Generators::WeightsGenerator.new self.inverted
         self.weights = generator.generate self.weights_strategy
       end
+      # Generates a new similarity index (writes its index) using the
+      # given similarity caching strategy.
+      #
+      def generate_similarity
+        generator = Generators::SimilarityGenerator.new self.inverted
+        self.similarity = generator.generate self.similarity_strategy
+      end
 
       # Saves the indexes in a dump file.
       #
       def dump
-        dump_index
+        timed_exclaim %Q{"#{identifier}": Dumping data.}
+        dump_inverted
         dump_similarity
         dump_weights
         dump_configuration
       end
       # Dumps the core index.
       #
-      def dump_index
-        timed_exclaim %Q{"#{identifier}": Dumping index.}
-        backend.dump_index index
+      def dump_inverted
+        # timed_exclaim %Q{"#{identifier}": Dumping inverted index.}
+        backend.dump_inverted self.inverted
       end
       # Dumps the weights index.
       #
       def dump_weights
-        timed_exclaim %Q{"#{identifier}": Dumping weights of index.}
-        backend.dump_weights weights
+        # timed_exclaim %Q{"#{identifier}": Dumping index weights.}
+        backend.dump_weights self.weights
       end
       # Dumps the similarity index.
       #
       def dump_similarity
-        timed_exclaim %Q{"#{identifier}": Dumping similarity of index.}
-        backend.dump_similarity similarity
+        # timed_exclaim %Q{"#{identifier}": Dumping similarity index.}
+        backend.dump_similarity self.similarity
       end
       # Dumps the similarity index.
       #
       def dump_configuration
-        timed_exclaim %Q{"#{identifier}": Dumping configuration for index.}
-        backend.dump_configuration configuration
+        # timed_exclaim %Q{"#{identifier}": Dumping configuration.}
+        backend.dump_configuration self.configuration
       end
 
       # Alerts the user if an index is missing.
@@ -195,14 +214,14 @@ module Indexing # :nodoc:all
       # Warns the user if the core or weights indexes are small.
       #
       def warn_if_index_small
-        warn_cache_small :index   if backend.index_cache_small?
-        warn_cache_small :weights if backend.weights_cache_small?
+        warn_cache_small :inverted if backend.inverted_cache_small?
+        warn_cache_small :weights  if backend.weights_cache_small?
       end
       # Alerts the user if the core or weights indexes are not there.
       #
       def raise_unless_index_ok
-        raise_cache_missing :index   unless backend.index_cache_ok?
-        raise_cache_missing :weights unless backend.weights_cache_ok?
+        raise_cache_missing :inverted unless backend.inverted_cache_ok?
+        raise_cache_missing :weights  unless backend.weights_cache_ok?
       end
 
     end

data/lib/picky/indexing/bundle/memory.rb

@@ -14,10 +14,6 @@ module Indexing # :nodoc:all
       #
       alias backend files
 
-      def to_s
-        "Memory\n#{@backend.indented_to_s}"
-      end
-
     end
 
   end

data/lib/picky/loader.rb

@@ -145,7 +145,7 @@ module Loader # :nodoc:all
 
     # Index store handling.
     #
-    load_relative 'backend/backend'
+    load_relative 'backend/base'
 
     load_relative 'backend/redis'
     load_relative 'backend/redis/basic'
@@ -158,10 +158,11 @@ module Loader # :nodoc:all
     load_relative 'backend/file/json'
 
     load_relative 'backend/files'
-    
+
     # Indexing and Indexed things.
     #
-    load_relative 'indexing/bundle/super_base' # TODO Remove.
+    load_relative 'bundle'
+
     load_relative 'indexing/bundle/base'
     load_relative 'indexing/bundle/memory'
     load_relative 'indexing/bundle/redis'
@@ -260,17 +261,17 @@ module Loader # :nodoc:all
     load_relative 'category'
     load_relative 'category_indexed'
     load_relative 'category_indexing'
-    
+
     load_relative 'categories'
     load_relative 'categories_indexed'
     load_relative 'categories_indexing'
-    
+
     load_relative 'index/base'
     load_relative 'index/base_indexed'
     load_relative 'index/base_indexing'
     load_relative 'index/memory'
     load_relative 'index/redis'
-    
+
     load_relative 'indexes'
     load_relative 'indexes_indexed'
     load_relative 'indexes_indexing'

data/lib/picky/query/allocation.rb

@@ -6,7 +6,7 @@ module Query
   class Allocation # :nodoc:all
 
     attr_reader :count, :ids, :score, :combinations, :result_identifier
-  
+
     #
     #
     def initialize combinations, result_identifier
@@ -76,9 +76,9 @@ module Query
     #
     #
     def to_s
-      "Allocation: #{to_result.join(', ')}"
+      "Allocation(#{to_result})"
     end
 
   end
-  
+
 end

data/lib/picky/query/token.rb

@@ -17,6 +17,8 @@ module Query
     #
     # Note: Use this if you do not want a qualified and normalized token.
     #
+    # TODO text, qualifiers
+    #
     def initialize text
       @text = text
     end
@@ -50,6 +52,8 @@ module Query
     #
     # Note: Removes the qualifier if it is not allowed.
     #
+    # TODO Extract this sind it is Search-based.
+    #
     def qualify
       @qualifiers, @text = split @text
       @qualifiers && @qualifiers.collect! { |qualifier| Query::Qualifiers.instance.normalize qualifier }.compact!
@@ -159,7 +163,7 @@ module Query
     # Internal identifier.
     #
     def identifier
-      "#{similar?? :similarity : :index}:#{@text}"
+      "#{similar?? :similarity : :inverted}:#{@text}"
     end
 
     # If the originals & the text are the same, they are the same.

data/lib/picky/search.rb

@@ -24,6 +24,7 @@ class Search
   #   * weights:   A hash of weights, or a Query::Weights object.
   #
   # TODO Add identifiers_to_remove (rename) and reduce_allocations_to_amount (rename).
+  # TODO categories_to_remove ?
   #
   # It is also possible to define the tokenizer and weights like so.
   # Example:
@@ -57,9 +58,13 @@ class Search
       options && Tokenizers::Query.new(options)
     end
   end
+
+  # Returns the tokenizer if set or if not, the query tokenizer.
+  #
   def tokenizer
     @tokenizer || Tokenizers::Query.default
   end
+
   # TODO Doc. Spec.
   #
   # Example:

data/lib/picky/sources/base.rb

@@ -18,7 +18,7 @@ module Sources
   # A source has 1 mandatory and 2 optional methods:
   # * connect_backend (_optional_): called once for each type/category pair.
   # * harvest: Used by the indexer to gather data. Yields an indexed_id (string or integer) and a string value.
-  # * take_snapshot (_optional_): called once for each type.
+  # * take_snapshot (_optional_): called once for each index or category (if indexing a single category).
   #
   # This base class "implements" all these methods, but they don't do anything.
   # Subclass this class <tt>class MySource < Base</tt> and override the methods in your source to do something.
@@ -56,7 +56,9 @@ module Sources
 
     # Used to take a snapshot of your data if it is fast changing.
     #
-    # Called once for each type before harvesting.
+    # Called once for each index before harvesting.
+    # If it has been called on a source already by an index,
+    # it won't be called again for a category inside that index.
     #
     # Example:
     # * In a DB source, a table based on the source's select statement is created.
@@ -65,6 +67,23 @@ module Sources
 
     end
 
+    # Used to check if a snapshot has been done already.
+    #
+    # Example:
+    # * In a DB source, a table based on the source's select statement is created.
+    #
+    def with_snapshot index
+      connect_backend
+      @snapshot_taken ||= 0
+      if @snapshot_taken.zero?
+        timed_exclaim %Q{"#{index.identifier}": Taking snapshot of source data (if supported).}
+        take_snapshot index
+      end
+      @snapshot_taken += 1
+      yield
+      @snapshot_taken -= 1
+    end
+
   end
 
 end

data/lib/picky/sources/db.rb

@@ -85,8 +85,6 @@ module Sources
     # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
     #
     def take_snapshot index
-      connect_backend
-
       origin = snapshot_table_name index.name
       on_database = database.connection
 
@@ -110,8 +108,6 @@ module Sources
     # Counts all the entries that are used for the index.
     #
     def count index_name
-      connect_backend
-
       database.connection.select_value("SELECT COUNT(#{@@traversal_id}) FROM #{snapshot_table_name(index_name)}").to_i
     end
 
@@ -124,8 +120,6 @@ module Sources
     # Harvests the data to index in chunks.
     #
     def harvest category, &block
-      connect_backend
-
       (0..count(category.index_name)).step(chunksize) do |offset|
         get_data category, offset, &block
       end
@@ -134,7 +128,6 @@ module Sources
     # Gets the data from the backend.
     #
     def get_data category, offset, &block # :nodoc:
-
       select_statement = harvest_statement_with_offset category, offset
 
       # TODO Rewrite ASAP.

data/lib/picky/statistics.rb

@@ -5,8 +5,8 @@
 #
 class Statistics # :nodoc:all
 
-  def self.instance
-    @statistics ||= new
+  def initialize
+    @indexes = ["\033[1mIndexes analysis\033[m:"]
   end
 
   def preamble
@@ -28,18 +28,15 @@ PREAMBLE
 
   # Gathers information about the indexes.
   #
-  def analyze
-    preamble
-
-    @indexes = ["\033[1mIndexes analysis\033[m:"]
-    Indexes.analyze.each_pair do |name, index|
+  def analyze object
+    object.each_category do |category|
       @indexes << <<-ANALYSIS
-#{"#{name}:".indented_to_s}:
-#{"exact:\n#{index[:exact].indented_to_s}".indented_to_s(4)}
-#{"partial*:\n#{index[:partial].indented_to_s}".indented_to_s(4)}
+#{"#{category.index_name}".indented_to_s}\n
+#{"#{category.name}".indented_to_s(4)}\n
+#{"exact\n#{Analyzer.new.analyze(category.indexed_exact).indented_to_s}".indented_to_s(6)}\n
+#{"partial\n#{Analyzer.new.analyze(category.indexed_partial).indented_to_s}".indented_to_s(6)}
 ANALYSIS
     end
-    @indexes = @indexes.join "\n"
   end
 
   # Outputs all gathered statistics.
@@ -49,7 +46,7 @@ ANALYSIS
 
 Picky Configuration:
 
-#{[@preamble, @application, @indexes].compact.join("\n")}
+#{[@preamble, @application, @indexes.join("\n")].compact.join("\n")}
 STATS
   end
 

data/lib/picky/tokenizers/location.rb

@@ -17,7 +17,7 @@ module Tokenizers
       @locations = []
     end
 
-    # TODO Work on this!
+    #
     #
     def tokenize text
 

data/lib/tasks/checks.rake

@@ -1,11 +1,13 @@
 # Checks to help the user.
 #
-namespace :check do
+desc 'Checks if index files are small/missing (index, category optional).'
+task :check, [:index, :category] => :application do |_, options|
+  index, category = options.index, options.category
 
-  desc 'Checks the index files for files that are suspiciously small or missing.'
-  task :index => :application do
-    Indexes.check_caches
-    puts "All indexes look ok."
-  end
+  specific = Indexes
+  specific = specific[index]    if index
+  specific = specific[category] if category
+  specific.check
 
+  puts "All checked indexes look ok."
 end

data/lib/tasks/index.rake

@@ -1,32 +1,26 @@
 # Indexing tasks.
 #
+desc "Generate the index (index, category optional)."
+task :index, [:index, :category] => :application do |_, options|
+  index, category = options.index, options.category
+
+  specific = Indexes
+  specific = specific[index]    if index
+  specific = specific[category] if category
+  specific.index
+end
+
 namespace :index do
 
-  desc "Takes a snapshot, indexes, and caches in random order."
+  # Advanced usage.
+  #
+  # desc "Takes a snapshot, indexes, and caches in random order."
   task :randomly => :application do
     Indexes.index true
   end
-  desc "Takes a snapshot, indexes, and caches in order given."
+  # desc "Takes a snapshot, indexes, and caches in order given."
   task :ordered => :application do
     Indexes.index false
   end
 
-  # desc "Generates the index snapshots."
-  #
-  # Note: Hidden since it is only needed by pro users.
-  #
-  # desc "Generate the data snapshots (intermediate table on a DB source)"
-  task :generate_snapshots => :application do
-    Indexes.take_snapshot
-  end
-
-  desc "Generates a specific index from index snapshots (category optional)."
-  task :specific, [:index, :category] => :application do |_, options|
-    index, category = options.index, options.category
-
-    specific = Indexes[index]
-    specific = specific[category] if category
-    specific.index
-  end
-
 end

data/lib/tasks/server.rake

@@ -1,5 +1,14 @@
 # Server tasks, like starting/stopping/restarting.
 #
+desc "Start the server."
+task :start do
+  Rake::Task[:'server:start'].invoke
+end
+desc "Stop the server."
+task :stop do
+  Rake::Task[:'server:stop'].invoke
+end
+
 namespace :server do
 
   # desc "Start the unicorns. (Wehee!)"
@@ -30,8 +39,15 @@ namespace :server do
   end
 
   def current_pid
-    pid = `cat #{File.join(PICKY_ROOT, 'tmp/pids/unicorn.pid')}`
-    pid.blank? ? nil : pid.chomp
+    pidfile = 'tmp/pids/unicorn.pid'
+    pid = `cat #{File.join(PICKY_ROOT, pidfile)}`
+    if pid.blank?
+      puts
+      puts "No server running (no #{pidfile} found)."
+      puts
+    else
+      pid.chomp
+    end
   end
 
 end

data/lib/tasks/statistics.rake

@@ -1,21 +1,34 @@
 # Statistics tasks.
 #
+desc "Analyzes indexes (index, category optional)."
+task :analyze, [:index, :category] => :'stats:prepare' do |_, options|
+  index, category = options.index, options.category
+
+  specific = Indexes
+  specific = specific[index]    if index
+  specific = specific[category] if category
+
+  statistics = Statistics.new
+
+  begin
+    statistics.analyze specific
+  rescue StandardError
+    puts "\n\033[31mNote: rake analyze needs prepared indexes. Run rake index first.\033[m\n\n"
+    raise
+  end
+
+  puts statistics
+end
+
+task :stats => :'stats:prepare' do
+  stats = Statistics.new
+  puts stats.application
+end
+
 namespace :stats do
+
   task :prepare => :application do
     require File.expand_path('../../picky/statistics', __FILE__)
-    statistics = Statistics.instance
-  end
-  task :app => :prepare do
-    Statistics.instance.application
-    puts Statistics.instance
-  end
-  task :analyze => :prepare do
-    begin
-      Statistics.instance.analyze
-    rescue StandardError
-      puts "\n\033[31mNote: rake analyze needs prepared indexes. Run rake index first.\033[m\n\n"
-      raise
-    end
-    puts Statistics.instance
   end
+
 end

data/lib/tasks/todo.rake

@@ -1,8 +1,8 @@
 desc "Finds where Picky still needs input from you."
 task :"to#{}do" do
-  if system "grep -e 'TODO.*' -n --color=always -R *"
+  if system "grep -e 'TO#{}DO.*' -n --color=always -R *"
     puts "Picky needs a bit of input from you there. Thanks."
   else
-    puts "Picky seems to be fine (no TODOs found)."
+    puts "Picky seems to be fine (no TO#{}DOs found)."
   end
 end