picky 2.6.0 → 2.7.0

data/lib/picky/analyzer.rb

@@ -17,11 +17,11 @@ class Analyzer # :nodoc:all
   #
   #
   def analyze bundle
-    bundle.load_index
-    analysis[:__keys] = bundle.size
-    cardinality :index, bundle.index
+    bundle.load_inverted
+    analysis[:__keys] = bundle.inverted.size
+    cardinality :index, bundle.inverted
     index_analysis
-    bundle.clear_index
+    bundle.clear_inverted
 
     bundle.load_weights
     weights bundle.weights

data/lib/picky/application.rb

@@ -68,7 +68,7 @@
 # In full glory:
 #   class MyGreatSearch < Application
 #
-#     books = index :books do
+#     books = Index::Memory.new :books do
 #       source   Sources::CSV.new(:title, :author, :isbn, file:'app/library.csv')
 #       category :title
 #     end
@@ -227,12 +227,6 @@ class Application
       "WARNING: No routes defined for application configuration in #{self.class}." if rack_adapter.empty?
     end
 
-    def to_s # :nodoc:
-      <<-APPLICATION
-\033[1m#{name}\033[m
-#{to_stats.indented_to_s}
-APPLICATION
-    end
     def to_stats
       <<-APP
 \033[1mIndexing (default)\033[m:
@@ -248,10 +242,15 @@ APPLICATION
 #{to_routes.indented_to_s}
 APP
     end
+
     def to_routes
       rack_adapter.to_s
     end
 
+    def to_s # :nodoc:
+      self.name
+    end
+
   end
 
 end

data/lib/picky/backend/{backend.rb → base.rb}

@@ -1,13 +1,20 @@
 module Backend
 
-  class Backend
+  class Base
 
-    attr_reader :bundle_name
-    attr_reader :prepared, :index, :weights, :similarity, :configuration
+    attr_reader :bundle_name,
+                :prepared,
+                :inverted,
+                :weights,
+                :similarity,
+                :configuration,
+                :category,
+                :identifier
 
     delegate :index_name, :category_name, :to => :@category
 
     def initialize bundle_name, category
+      @identifier  = "#{category.identifier}:#{bundle_name}"
       @bundle_name = bundle_name
       @category    = category
       @prepared    = File::Text.new category.prepared_index_path
@@ -24,23 +31,27 @@ module Backend
 
     # Dumping.
     #
-    def dump_index index_hash
-      index.dump index_hash
+    def dump_inverted inverted_hash
+      timed_exclaim %Q{"#{identifier}":   => #{inverted}.}
+      inverted.dump inverted_hash
     end
     def dump_weights weights_hash
+      timed_exclaim %Q{"#{identifier}":   => #{weights}.}
       weights.dump weights_hash
     end
     def dump_similarity similarity_hash
+      timed_exclaim %Q{"#{identifier}":   => #{similarity}.}
       similarity.dump similarity_hash
     end
     def dump_configuration configuration_hash
+      timed_exclaim %Q{"#{identifier}":   => #{configuration}.}
       configuration.dump configuration_hash
     end
 
     # Loading.
     #
-    def load_index
-      index.load
+    def load_inverted
+      inverted.load
     end
     def load_similarity
       similarity.load
@@ -54,8 +65,8 @@ module Backend
 
     # Cache ok?
     #
-    def index_cache_ok?
-      index.cache_ok?
+    def inverted_cache_ok?
+      inverted.cache_ok?
     end
     def similarity_cache_ok?
       similarity.cache_ok?
@@ -66,8 +77,8 @@ module Backend
 
     # Cache small?
     #
-    def index_cache_small?
-      index.cache_small?
+    def inverted_cache_small?
+      inverted.cache_small?
     end
     def similarity_cache_small?
       similarity.cache_small?
@@ -79,7 +90,7 @@ module Backend
     # Copies the indexes to the "backup" directory.
     #
     def backup
-      index.backup
+      inverted.backup
       weights.backup
       similarity.backup
       configuration.backup
@@ -88,7 +99,7 @@ module Backend
     # Restores the indexes from the "backup" directory.
     #
     def restore
-      index.restore
+      inverted.restore
       weights.restore
       similarity.restore
       configuration.restore
@@ -97,12 +108,18 @@ module Backend
     # Delete all index files.
     #
     def delete
-      index.delete
+      inverted.delete
       weights.delete
       similarity.delete
       configuration.delete
     end
 
+    #
+    #
+    def to_s
+      "#{self.class}(#{prepared}, #{bundle_name}, #{category.identifier})"
+    end
+
   end
 
 end

data/lib/picky/backend/file/basic.rb

@@ -13,6 +13,8 @@ module Backend
     #
     class Basic
 
+      # This file's location.
+      #
       attr_reader :cache_path
 
       # An index cache takes a path, without file extension,
@@ -22,10 +24,6 @@ module Backend
         @cache_path = "#{cache_path}.#{extension}"
       end
 
-      def to_s
-        cache_path
-      end
-
       # The default extension for index files is "index".
       #
       def extension
@@ -40,12 +38,15 @@ module Backend
         prepare_backup backup_directory
         FileUtils.cp cache_path, target, verbose: true
       end
+
       # The backup directory of this file.
       # Equal to the file's dirname plus /backup
       #
+
       def backup_directory
         ::File.join ::File.dirname(cache_path), 'backup'
       end
+
       # Prepares the backup directory for the file.
       #
       def prepare_backup target
@@ -58,6 +59,7 @@ module Backend
       def restore
         FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
       end
+
       # The backup filename.
       #
       def backup_file_path_of path
@@ -94,6 +96,12 @@ module Backend
         `ls -l #{path} | awk '{print $5}'`.to_i
       end
 
+      #
+      #
+      def to_s
+        "#{self.class}(#{cache_path})"
+      end
+
     end
 
   end

data/lib/picky/backend/file/json.rb

@@ -1,11 +1,11 @@
 module Backend
-  
+
   module File
-  
+
     # Index files dumped in the JSON format.
     #
     class JSON < Basic
-    
+
       # Uses the extension "json".
       #
       def extension
@@ -26,9 +26,9 @@ module Backend
       def retrieve
         raise "Can't retrieve from JSON file. Use text file."
       end
-    
+
     end
-  
+
   end
 
 end

data/lib/picky/backend/file/text.rb

@@ -44,7 +44,7 @@ module Backend
 
       #
       #
-      def open_for_indexing &block
+      def open &block
         ::File.open cache_path, 'w:binary', &block
       end
 

data/lib/picky/backend/files.rb

@@ -1,6 +1,6 @@
 module Backend
 
-  class Files < Backend
+  class Files < Base
 
     def initialize bundle_name, category
       super bundle_name, category
@@ -9,20 +9,14 @@ module Backend
       #       Yajl json lib cannot load symbolized
       #       values, just keys.
       #
-      @index         = File::JSON.new    category.index_path(bundle_name, :index)
+      @inverted      = File::JSON.new    category.index_path(bundle_name, :inverted)
       @weights       = File::JSON.new    category.index_path(bundle_name, :weights)
       @similarity    = File::Marshal.new category.index_path(bundle_name, :similarity)
       @configuration = File::JSON.new    category.index_path(bundle_name, :configuration)
     end
 
     def to_s
-      <<-FILES
-Files:
-#{"Index:      #{@index}".indented_to_s}
-#{"Weights:    #{@weights}".indented_to_s}
-#{"Similarity: #{@similarity}".indented_to_s}
-#{"Config:     #{@configuration}".indented_to_s}
-FILES
+      "#{self.class}(#{[@prepared, @inverted, @weights, @similarity, @configuration].join(', ')})"
     end
 
   end

data/lib/picky/backend/redis/basic.rb

@@ -61,6 +61,7 @@ module Backend
       def cache_small?
         size < 1
       end
+
       # Is the cache ok?
       #
       # A small cache is still ok.
@@ -68,6 +69,7 @@ module Backend
       def cache_ok?
         size > 0
       end
+
       # Extracts the size of the file in Bytes.
       #
       # Note: This is a very forgiving implementation.
@@ -78,6 +80,12 @@ module Backend
         backend.dbsize
       end
 
+      #
+      #
+      def to_s
+        "#{self.class}(#{namespace}:*)"
+      end
+
     end
 
   end

data/lib/picky/backend/redis/list_hash.rb

@@ -17,7 +17,7 @@ module Backend
           end
         end
       end
-      
+
       # Clear the index for this list.
       #
       # Note: Perhaps we can use a server only command.
@@ -32,14 +32,14 @@ module Backend
 
       # Get a collection.
       #
-      def collection sym
-        backend.lrange "#{namespace}:#{sym}", 0, -1
+      def collection key
+        backend.zrange "#{namespace}:#{key}", 0, -1
       end
 
       # Get a single value.
       #
-      def member sym
-        raise "Can't retrieve a single value from a Redis ListHash. Use Index::Redis::StringHash."
+      def member key
+        raise "Can't retrieve single value :#{key} from a Redis ListHash. Use Index::Redis::StringHash."
       end
 
     end

data/lib/picky/backend/redis/string_hash.rb

@@ -14,7 +14,7 @@ module Backend
           backend.hset namespace, key, value
         end
       end
-      
+
       # Clears the hash.
       #
       def clear
@@ -23,14 +23,14 @@ module Backend
 
       # Get a collection.
       #
-      def collection sym
-        raise "Can't retrieve a collection from a StringHash. Use Index::Redis::ListHash."
+      def collection key
+        raise "Can't retrieve collection for :#{key} from a StringHash. Use Index::Redis::ListHash."
       end
 
       # Get a single value.
       #
-      def member sym
-        backend.hget namespace, sym
+      def member key
+        backend.hget namespace, key
       end
 
     end

data/lib/picky/backend/redis.rb

@@ -1,24 +1,24 @@
 module Backend
 
-  # TODO Needs a reconnect to be run after forking.
   #
-  class Redis < Backend
+  #
+  class Redis < Base
 
     def initialize bundle_name, category
       super bundle_name, category
 
       # Refine a few Redis "types".
       #
-      @index         = Redis::ListHash.new "#{category.identifier}:#{bundle_name}:index"
+      @inverted      = Redis::ListHash.new   "#{category.identifier}:#{bundle_name}:inverted"
       @weights       = Redis::StringHash.new "#{category.identifier}:#{bundle_name}:weights"
-      @similarity    = Redis::ListHash.new "#{category.identifier}:#{bundle_name}:similarity"
+      @similarity    = Redis::ListHash.new   "#{category.identifier}:#{bundle_name}:similarity"
       @configuration = Redis::StringHash.new "#{category.identifier}:#{bundle_name}:configuration"
     end
 
     # Delegate to the right collection.
     #
     def ids sym
-      index.collection sym
+      inverted.collection sym
     end
 
     # Delegate to the right member value.

data/lib/picky/bundle.rb

@@ -0,0 +1,62 @@
+# A Bundle is a number of indexes
+# per [index, category] combination.
+#
+# At most, there are three indexes:
+# * *core* index (always used)
+# * *weights* index (always used)
+# * *similarity* index (used with similarity)
+#
+# In Picky, indexing is separated from the index
+# handling itself through a parallel structure.
+#
+# Both use methods provided by this base class, but
+# have very different goals:
+#
+# * *Indexing*::*Bundle*::*Base* is just concerned with creating index
+#   files / redis entries and providing helper functions to e.g. check
+#   the indexes.
+#
+# * *Index*::*Bundle*::*Base* is concerned with loading these index files into
+#   memory / redis and looking up search data as fast as possible.
+#
+class Bundle
+
+  attr_reader   :identifier,
+                :files
+  attr_accessor :inverted,
+                :weights,
+                :similarity,
+                :configuration,
+                :similarity_strategy
+
+  delegate :clear,    :to => :inverted
+  delegate :[], :[]=, :to => :configuration
+
+  def initialize name, category, similarity_strategy
+    @identifier    = "#{category.identifier}:#{name}"
+    @files         = Backend::Files.new name, category
+
+    @inverted      = {}
+    @weights       = {}
+    @similarity    = {}
+    @configuration = {} # A hash with config options.
+
+    @similarity_strategy = similarity_strategy
+  end
+
+  # Get a list of similar texts.
+  #
+  # Note: Does not return itself.
+  #
+  def similar text
+    code = similarity_strategy.encoded text
+    similar_codes = code && @similarity[code]
+    similar_codes.delete text if similar_codes
+    similar_codes || []
+  end
+
+  def to_s
+    "#{self.class}(#{identifier}, #{files})"
+  end
+
+end

data/lib/picky/categories.rb

@@ -8,6 +8,7 @@ class Categories
            :to => :categories
 
   each_delegate :reindex,
+                :each_category,
                 :to => :categories
 
   # A list of indexed categories.
@@ -26,18 +27,25 @@ class Categories
   #                              Nifty! :)
   #
   def initialize options = {}
-    clear
+    clear_categories
 
     @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
   end
 
   # Clears both the array of categories and the hash of categories.
   #
-  def clear
+  def clear_categories
     @categories    = []
     @category_hash = {}
   end
 
+  # Add the given category to the list of categories.
+  #
+  def << category
+    categories << category
+    category_hash[category.name] = category
+  end
+
   # Find a given category in the categories.
   #
   def [] category_name
@@ -48,13 +56,6 @@ class Categories
     raise %Q{Index category "#{category_name}" not found. Possible categories: "#{categories.map(&:name).join('", "')}".}
   end
 
-  # Add the given category to the list of categories.
-  #
-  def << category
-    categories << category
-    category_hash[category.name] = category
-  end
-
   def to_s
     categories.join(', ')
   end

data/lib/picky/categories_indexed.rb

@@ -1,5 +1,5 @@
 class Categories
- 
+
   attr_reader :ignore_unassigned_tokens
 
   each_delegate :load_from_cache,
@@ -15,17 +15,17 @@ class Categories
   def possible_combinations_for token
     token.similar? ? similar_possible_for(token) : possible_for(token)
   end
+
   # Gets all similar tokens and puts together the possible combinations
   # for each found similar token.
   #
   def similar_possible_for token
-    # Get as many tokens as necessary
-    #
     tokens = similar_tokens_for token
-    # possible combinations
-    #
     inject_possible_for tokens
   end
+
+  # Returns all possible similar tokens for the given token.
+  #
   def similar_tokens_for token
     text = token.text
     categories.inject([]) do |result, category|
@@ -41,6 +41,9 @@ class Categories
       result
     end
   end
+
+  #
+  #
   def inject_possible_for tokens
     tokens.inject([]) do |result, token|
       possible = possible_categories token
@@ -65,8 +68,9 @@ class Categories
     # This is an optimization to mark tokens that are ignored.
     #
     return if ignore_unassigned_tokens && possible.empty?
-    possible # wrap in combinations
+    possible
   end
+
   # This returns the possible categories for this token.
   # If the user has already preselected a category for this token,
   # like "artist:moby", if not just return all for the given token,
@@ -77,6 +81,7 @@ class Categories
   def possible_categories token
     user_defined_categories(token) || categories
   end
+
   # This returns the array of categories if the user has defined
   # an existing category.
   #
@@ -89,5 +94,5 @@ class Categories
       category_hash[name]
     end.compact
   end
-  
+
 end

data/lib/picky/categories_indexing.rb

@@ -1,12 +1,10 @@
 class Categories
-  
-  each_delegate :backup_caches,
-                :cache,
-                :check_caches,
-                :clear_caches,
-                :create_directory_structure,
-                :generate_caches,
-                :restore_caches,
+
+  each_delegate :cache,
+                :check,
+                :clear,
+                :backup,
+                :restore,
                 :to => :categories
-  
+
 end