picky 0.9.0 → 0.9.1

data/lib/picky/cacher/partial/default.rb

@@ -1,5 +1,5 @@
 module Cacher
   module Partial
-    Default = Subtoken.new :down_to => -3, :starting_at => -1
+    Default = Substring.new :from => -3, :to => -1
   end
 end

data/lib/picky/cacher/partial/subtoken.rb

@@ -4,20 +4,20 @@ module Cacher
     
     # Generates the right subtokens for use in the subtoken strategy.
     #
-    class SubtokenGenerator
+    class SubstringGenerator
       
-      attr_reader :down_to, :starting_at
+      attr_reader :from, :to
       
-      def initialize down_to, starting_at
-        @down_to, @starting_at = down_to, starting_at
+      def initialize from, to
+        @from, @to = from, to
         
-        if @starting_at.zero?
+        if @to.zero?
           def each_subtoken token, &block
-            token.each_subtoken @down_to, &block
+            token.each_subtoken @from, &block
           end
         else
           def each_subtoken token, &block
-            token[0..@starting_at].intern.each_subtoken @down_to, &block
+            token[0..@to].intern.each_subtoken @from, &block
           end
         end
         
@@ -36,31 +36,31 @@ module Cacher
     # "flo"
     # "fl"
     # "f"
-    # Depending on what the given down_to value is. (Example with down_to == 1)
+    # Depending on what the given from value is. (Example with from == 1)
     #
-    class Subtoken < Strategy
+    class Substring < Strategy
       
       # Down to is how far it will go down in generating the subtokens.
       #
       # Examples:
-      # With :hello, and starting_at -1
+      # With :hello, and to -1
       # * down to == 1: [:hello, :hell, :hel, :he, :h]
       # * down to == 4: [:hello, :hell]
       #
-      # With :hello, and starting_at -2
+      # With :hello, and to -2
       # * down to == 1: [:hell, :hel, :he, :h]
       # * down to == 4: [:hell]
       #
       def initialize options = {}
-        down_to     = options[:down_to] || 1
-        starting_at = options[:starting_at] || -1
-        @generator = SubtokenGenerator.new down_to, starting_at
+        from     = options[:from] || 1
+        to = options[:to] || -1
+        @generator = SubstringGenerator.new from, to
       end
-      def down_to
-        @generator.down_to
+      def from
+        @generator.from
       end
-      def starting_at
-        @generator.starting_at
+      def to
+        @generator.to
       end
       
       # Generates a partial index from the given index.

data/lib/picky/cacher/partial_generator.rb

@@ -6,7 +6,7 @@ module Cacher
     
     # Generate a partial index based on the given index.
     #
-    def generate strategy = Partial::Subtoken.new(:down_to => 1)
+    def generate strategy = Partial::Substring.new(:from => 1)
       strategy.generate_from self.index
     end
 

data/lib/picky/extensions/hash.rb

@@ -4,16 +4,16 @@ class Hash
   
   # Dumps jsonized self to the path given. Minus extension.
   #
-  def dump_to_json path
-    File.open("#{path}.json", 'w') do |out_file|
+  def dump_json path
+    File.open(path, 'w') do |out_file|
       Yajl::Encoder.encode self, out_file
     end
   end
   
   # Dumps binary self to the path given. Minus extension.
   #
-  def dump_to_marshalled path
-    File.open("#{path}.dump", 'w:binary') do |out_file|
+  def dump_marshalled path
+    File.open(path, 'w:binary') do |out_file|
       Marshal.dump self, out_file
     end
   end

data/lib/picky/extensions/symbol.rb

@@ -5,30 +5,30 @@ class Symbol
   # :keys.subtokens    # => [:keys, :key, :ke, :k]
   # :keys.subtokens(2) # => [:keys, :key, :ke]
   #
-  def subtokens down_to_length = 1
+  def subtokens from_length = 1
     sub = self.id2name
     
     size = sub.size
-    down_to_length = size + down_to_length if down_to_length < 0
-    down_to_length = size if size < down_to_length
+    from_length = size + from_length if from_length < 0
+    from_length = size if size < from_length
     
     result = [self]
-    size.downto(down_to_length + 1) { result << sub.chop!.intern }
+    size.downto(from_length + 1) { result << sub.chop!.intern }
     result
   end
   
   # TODO Duplicate code.
   #
-  def each_subtoken down_to_length = 1
+  def each_subtoken from_length = 1
     sub = self.id2name
     
     size = sub.size
-    down_to_length = size + down_to_length + 1 if down_to_length < 0
-    down_to_length = size if size < down_to_length
-    down_to_length = 1 if down_to_length < 1
+    from_length = size + from_length + 1 if from_length < 0
+    from_length = size if size < from_length
+    from_length = 1 if from_length < 1
     
     yield self
-    size.downto(down_to_length + 1) { yield sub.chop!.intern }
+    size.downto(from_length + 1) { yield sub.chop!.intern }
   end
   
 end

data/lib/picky/index/bundle.rb

@@ -6,12 +6,14 @@ module Index
   #
   # Handles exact index, partial index, weights index, and similarity index.
   #
+  # Delegates file handling and checking to a Index::Files object.
+  #
   class Bundle
     
-    attr_reader   :checker
-    attr_reader   :name,             :category,         :type
+    attr_reader   :identifier, :category
     attr_accessor :index,            :weights,          :similarity
     attr_accessor :partial_strategy, :weights_strategy, :similarity_strategy
+    attr_reader   :files
     
     delegate :[], :[]=, :clear, :to => :index
     delegate :raise_unless_cache_exists, :to => :checker
@@ -19,19 +21,21 @@ module Index
     # Path is in which directory the cache is located.
     #
     def initialize name, category, type, partial_strategy, weights_strategy, similarity_strategy
+      @identifier = "#{name}: #{type.name} #{category.name}"
+      
       @index      = {}
       @weights    = {}
       @similarity = {}
       
-      @name     = name
+      # TODO Used in weights, try to remove!
+      #
       @category = category
-      @type     = type
       
       @partial_strategy    = partial_strategy
       @weights_strategy    = weights_strategy
       @similarity_strategy = similarity_strategy
       
-      @checker = BundleChecker.new self
+      @files   = Files.new name, category.name, type.name
     end
     
     # Get the ids for the text.
@@ -50,109 +54,6 @@ module Index
       code = similarity_strategy.encoded text
       code && @similarity[code] || []
     end
-
-    # Identifier for this bundle.
-    #
-    def identifier
-      "#{name}: #{type.name} #{category.name}"
-    end
-    
-    # Point to category.
-    #
-    def search_index_root
-      File.join PICKY_ROOT, 'index'
-      # category.search_index_root
-    end
-
-    # Copies the indexes to the "backup" directory.
-    #
-    def backup
-      target = backup_path
-      FileUtils.mkdir target unless Dir.exists?(target)
-      FileUtils.cp index_cache_path,      target, :verbose => true
-      FileUtils.cp similarity_cache_path, target, :verbose => true
-      FileUtils.cp weights_cache_path,    target, :verbose => true
-    end
-    def backup_path
-      File.join File.dirname(index_cache_path), 'backup'
-    end
-
-    # Restores the indexes from the "backup" directory.
-    #
-    def restore
-      FileUtils.cp backup_file_path_of(index_cache_path), index_cache_path, :verbose => true
-      FileUtils.cp backup_file_path_of(similarity_cache_path), similarity_cache_path, :verbose => true
-      FileUtils.cp backup_file_path_of(weights_cache_path), weights_cache_path, :verbose => true
-    end
-    def backup_file_path_of path
-      dir, name = File.split path
-      File.join dir, 'backup', name
-    end
-
-    # Delete the file at path.
-    #
-    def delete path
-      `rm -Rf #{path}`
-    end
-    # Delete all index files.
-    #
-    def delete_all
-      delete index_cache_path
-      delete similarity_cache_path
-      delete weights_cache_path
-    end
-
-    # Create directory and parent directories.
-    #
-    def create_directory
-      FileUtils.mkdir_p cache_directory
-    end
-    # TODO Move to config. Duplicate Code in field.rb.
-    #
-    def cache_directory
-      File.join search_index_root, PICKY_ENVIRONMENT, type.name.to_s
-    end
-
-    # Generates a cache path.
-    #
-    def cache_path text
-      File.join cache_directory, "#{name}_#{text}"
-    end
-    def index_cache_path
-      cache_path "#{category.name}_index"
-    end
-    def similarity_cache_path
-      cache_path "#{category.name}_similarity"
-    end
-    def weights_cache_path
-      cache_path "#{category.name}_weights"
-    end
-    
-    # Loads all indexes into this category.
-    #
-    def load
-      load_index
-      load_similarity
-      load_weights
-    end
-    def load_the_json path
-      Yajl::Parser.parse File.open("#{path}.json", 'r'), :symbolize_keys => true
-    end
-    def load_the_marshalled path
-      Marshal.load File.open("#{path}.dump", 'r:binary')
-    end
-    def load_index
-      timed_exclaim "Loading the index for #{identifier} from the cache."
-      self.index = load_the_json index_cache_path
-    end
-    def load_similarity
-      timed_exclaim "Loading the similarity for #{identifier} from the cache."
-      self.similarity = load_the_marshalled similarity_cache_path
-    end
-    def load_weights
-      timed_exclaim "Loading the weights for #{identifier} from the cache."
-      self.weights = load_the_json weights_cache_path
-    end
     
     # Generation
     #
@@ -200,29 +101,17 @@ module Index
     # TODO Beautify.
     #
     def retrieve
-      # TODO Make r:binary configurable!
-      #
-      File.open(search_index_file_name, 'r:binary') do |file|
-        file.each_line do |line|
-          indexed_id, token = line.split ?,,2
-          token.chomp!
-          token = token.to_sym
-          
-          initialize_index_for token
-          index[token] << indexed_id.to_i
-        end
+      files.retrieve do |indexed_id, token|
+        token.chomp!
+        token = token.to_sym
+        
+        initialize_index_for token
+        index[token] << indexed_id.to_i
       end
     end
     def initialize_index_for token
       index[token] ||= []
     end
-    # TODO Duplicate code!
-    #
-    # TODO Use config object?
-    #
-    def search_index_file_name
-      File.join cache_directory, "prepared_#{category.name}_index.txt"
-    end
     
     # Generators.
     #
@@ -266,20 +155,56 @@ module Index
     end
     def dump_index
       timed_exclaim "DUMP INDEX #{identifier}."
-      index.dump_to_json index_cache_path
+      files.dump_index index
     end
-    # Note: We marshal the similarity, as the
-    #       Yajl json lib cannot load symbolized
-    #       values, just keys.
-    #
     def dump_similarity
       timed_exclaim "DUMP SIMILARITY #{identifier}."
-      similarity.dump_to_marshalled similarity_cache_path
+      files.dump_similarity similarity
     end
     def dump_weights
       timed_exclaim "DUMP WEIGHTS #{identifier}."
-      weights.dump_to_json weights_cache_path
+      files.dump_weights weights
+    end
+    
+    # Loads all indexes into this category.
+    #
+    def load
+      load_index
+      load_similarity
+      load_weights
+    end
+    def load_index
+      timed_exclaim "Loading the index for #{identifier} from the cache."
+      self.index = files.load_index
+    end
+    def load_similarity
+      timed_exclaim "Loading the similarity for #{identifier} from the cache."
+      self.similarity = files.load_similarity
     end
+    def load_weights
+      timed_exclaim "Loading the weights for #{identifier} from the cache."
+      self.weights = files.load_weights
+    end
+    
+    # Alerts the user if an index is missing.
+    #
+    def raise_unless_cache_exists
+      warn_cache_small :index      if files.index_cache_small?
+      warn_cache_small :similarity if files.similarity_cache_small?
+      warn_cache_small :weights    if files.weights_cache_small?
 
+      raise_cache_missing :index      unless files.index_cache_ok?
+      raise_cache_missing :similarity unless files.similarity_cache_ok?
+      raise_cache_missing :weights    unless files.weights_cache_ok?
+    end
+    def warn_cache_small what
+      puts "#{what} cache for #{identifier} smaller than 16 bytes."
+    end
+    # Raises an appropriate error message.
+    #
+    def raise_cache_missing what
+      raise "#{what} cache for #{identifier} missing."
+    end
+    
   end
 end

data/lib/picky/index/file/basic.rb

@@ -0,0 +1,67 @@
+module Index
+  
+  module File
+    
+    class Basic
+      
+      attr_reader :cache_path
+      
+      def initialize cache_path
+        @cache_path = "#{cache_path}.#{extension}"
+      end
+      
+      def extension
+        :index
+      end
+      
+      # Backup.
+      #
+      def backup
+        prepare_backup backup_path
+        FileUtils.cp cache_path, target, :verbose => true
+      end
+      def backup_path
+        ::File.join ::File.dirname(cache_path), 'backup'
+      end
+      def prepare_backup target
+        FileUtils.mkdir target unless Dir.exists?(target)
+      end
+      
+      # Restore.
+      #
+      def restore
+        FileUtils.cp backup_file_path_of(cache_path), cache_path, :verbose => true
+      end
+      def backup_file_path_of path
+        dir, name = ::File.split path
+        ::File.join dir, 'backup', name
+      end
+      
+      # Delete.
+      #
+      def delete
+        `rm -Rf #{cache_path}`
+      end
+      
+      # Checks.
+      #
+      
+      # Is the cache small?
+      #
+      def cache_small?
+        size_of(cache_path) < 16
+      end
+      # Is the cache ok? I.e. larger than four Bytes in size.
+      #
+      def cache_ok?
+        size_of(cache_path) > 0
+      end
+      def size_of path
+        `ls -l #{path} | awk '{print $5}'`.to_i
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/index/file/json.rb

@@ -0,0 +1,24 @@
+module Index
+  
+  module File
+    
+    class JSON < Basic
+      
+      def extension
+        :json
+      end
+      def load
+        Yajl::Parser.parse ::File.open(cache_path, 'r'), :symbolize_keys => true
+      end
+      def dump hash
+        hash.dump_json cache_path
+      end
+      def retrieve
+        raise "Can't retrieve from marshalled file. Use text file."
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/index/file/marshal.rb

@@ -0,0 +1,24 @@
+module Index
+  
+  module File
+    
+    class Marshal < Basic
+      
+      def extension
+        :dump
+      end
+      def load
+        ::Marshal.load ::File.open(cache_path, 'r:binary')
+      end
+      def dump hash
+        hash.dump_marshalled cache_path
+      end
+      def retrieve
+        raise "Can't retrieve from marshalled file. Use text file."
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/index/file/text.rb

@@ -0,0 +1,28 @@
+module Index
+  
+  module File
+    
+    class Text < Basic
+      
+      def extension
+        :txt
+      end
+      def load
+        raise "Can't load from text file. Use JSON or Marshal."
+      end
+      def dump hash
+        raise "Can't dump to text file. Use JSON or Marshal."
+      end
+      def retrieve
+        ::File.open(cache_path, 'r:binary') do |file|
+          file.each_line do |line|
+            yield line.split ?,, 2
+          end
+        end
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/picky/index/files.rb

@@ -0,0 +1,135 @@
+module Index
+  
+  # TODO Think about using 3 instances of this in the bundle.
+  #
+  class Files
+    
+    attr_reader :bundle_name, :category_name, :type_name
+    attr_reader :prepared, :index, :similarity, :weights
+    
+    def initialize bundle_name, category_name, type_name
+      @bundle_name   = bundle_name
+      @category_name = category_name
+      @type_name     = type_name
+      
+      # Note: We marshal the similarity, as the
+      #       Yajl json lib cannot load symbolized
+      #       values, just keys.
+      #
+      @prepared   = File::Text.new    "#{cache_directory}/prepared_#{category_name}_index"
+      @index      = File::JSON.new    cache_path(:index)
+      @similarity = File::Marshal.new cache_path(:similarity)
+      @weights    = File::JSON.new    cache_path(:weights)
+    end
+    
+    # Paths.
+    #
+    
+    # Cache path, for File-s.
+    #
+    def cache_path name
+      ::File.join cache_directory, "#{bundle_name}_#{category_name}_#{name}"
+    end
+    
+    # Point to category.
+    #
+    def search_index_root
+      ::File.join PICKY_ROOT, 'index'
+    end
+    
+    # Create directory and parent directories.
+    #
+    def create_directory
+      FileUtils.mkdir_p cache_directory
+    end
+    # TODO Move to config. Duplicate Code in field.rb.
+    #
+    def cache_directory
+      "#{search_index_root}/#{PICKY_ENVIRONMENT}/#{type_name}"
+    end
+    def retrieve &block
+      prepared.retrieve &block
+    end
+    
+    # Single index/similarity/weights files delegation.
+    #
+    
+    # Delegators.
+    #
+    
+    # Dumping.
+    #
+    def dump_index index_hash
+      index.dump index_hash
+    end
+    def dump_similarity similarity_hash
+      similarity.dump similarity_hash
+    end
+    def dump_weights weights_hash
+      weights.dump weights_hash
+    end
+    
+    # Loading.
+    #
+    def load_index
+      index.load
+    end
+    def load_similarity
+      similarity.load
+    end
+    def load_weights
+      weights.load
+    end
+    
+    # Cache ok?
+    #
+    def index_cache_ok?
+      index.cache_ok?
+    end
+    def similarity_cache_ok?
+      similarity.cache_ok?
+    end
+    def weights_cache_ok?
+      weights.cache_ok?
+    end
+    
+    # Cache small?
+    #
+    def index_cache_small?
+      index.cache_small?
+    end
+    def similarity_cache_small?
+      similarity.cache_small?
+    end
+    def weights_cache_small?
+      weights.cache_small?
+    end
+    
+    # Copies the indexes to the "backup" directory.
+    #
+    def backup
+      index.backup
+      similarity.backup
+      weights.backup
+    end
+    
+    # Restores the indexes from the "backup" directory.
+    #
+    def restore
+      index.restore
+      similarity.restore
+      weights.restore
+    end
+    
+    
+    # Delete all index files.
+    #
+    def delete
+      index.delete
+      similarity.delete
+      weights.delete
+    end
+    
+  end
+  
+end

data/lib/picky/indexers/base.rb

@@ -19,6 +19,8 @@ module Indexers
     end
     # Convenience methods for user subclasses.
     #
+    # TODO Duplicate code in Index::Files.
+    #
     def search_index_file_name
       @field.search_index_file_name
     end
@@ -48,6 +50,8 @@ module Indexers
       
       indexing_message
       
+      # TODO Move open to Index::File. 
+      #
       File.open(search_index_file_name, 'w:binary') do |file|
         result = []
         source.harvest(@type, @field) do |indexed_id, text|