picky 0.12.1 → 0.12.2

data/lib/picky/Index_api.rb

@@ -35,13 +35,20 @@ class IndexAPI
   end
   alias category define_category
   
+  # TODO Rewrite wrap_exact, wrap_source ?
+  #
   def define_location name, options = {}
-    grid      = options[:grid]
+    grid      = options[:grid] || raise("Grid size needs to be given to a location")
     precision = options[:precision]
     
-    define_category name, options do |indexing, _|
-      indexing.source    = Sources::Wrappers::Location.new indexing.source, grid: grid, precision: precision
+    define_category name, options do |indexing, indexed|
+      indexing.source    = Sources::Wrappers::Location.new indexing, grid: grid, precision: precision
       indexing.tokenizer = Tokenizers::Index.new
+      # indexing.partial = Partial::None.new
+      
+      exact_bundle    = Indexed::Wrappers::Bundle::Location.new indexed.exact, grid: grid
+      indexed.exact   = exact_bundle
+      indexed.partial = exact_bundle
     end
   end
   alias location define_location

data/lib/picky/cacher/partial/substring.rb

@@ -83,8 +83,8 @@ module Cacher
         
         # Remove duplicate ids.
         #
-        # TODO If it is unique for a subtoken, it is
-        #      unique for all derived longer tokens.
+        # THINK If it is unique for a subtoken, it is
+        #       unique for all derived longer tokens.
         #
         result.each_value &:uniq!
         
@@ -99,14 +99,14 @@ module Cacher
         #
         # "token" here means just text.
         #
-        # TODO Could be improved by appending the aforegoing ids?
+        # THINK Could be improved by appending the aforegoing ids?
         #
         def generate_for token, index, result
           @generator.each_subtoken(token) do |subtoken|
             if result[subtoken]
               result[subtoken] += index[token] # unique
             else
-              result[subtoken] = index[token].dup # TODO Spec this dup
+              result[subtoken] = index[token].dup
             end
           end
         end

data/lib/picky/calculations/location.rb

@@ -0,0 +1,39 @@
+module Calculations
+  
+  # A location calculation recalculates a 1-d location
+  # to the Picky internal 1-d "grid".
+  #
+  # For example, if you have a location x == 12.3456,
+  # it will be recalculated into 3, if the minimum is 9
+  # and the gridlength is 1.
+  #
+  class Location
+    
+    attr_reader :minimum
+    
+    def initialize user_grid, precision = nil
+      @user_grid = user_grid
+      @precision = precision || 1
+      @grid      = @user_grid / (@precision + 0.5)
+    end
+    
+    def minimum= minimum
+      minimum -= @user_grid
+      @minimum = minimum
+    end
+    
+    #
+    #
+    def add_margin length
+      @minimum -= length
+    end
+    
+    #
+    #
+    def recalculate location
+      ((location - @minimum) / @grid).floor
+    end
+    
+  end
+  
+end

data/lib/picky/configuration/index.rb

@@ -10,8 +10,8 @@ module Configuration
     attr_reader :index, :category
     
     def initialize index, category
-      @index    = index
-      @category = category
+      @index       = index
+      @category    = category
     end
     
     def index_name

data/lib/picky/extensions/array.rb

@@ -28,14 +28,6 @@ class Array
     result
   end
 
-  # Accesses a random element of this array.
-  #
-  # TODO Remove?
-  #
-  def random
-    self[Kernel.rand(self.length)]
-  end
-
   # Sort the array using distance from levenshtein.
   #
   # Will raise if encounters not to_s-able element.

data/lib/picky/extensions/symbol.rb

@@ -2,22 +2,8 @@
 #
 class Symbol
   
-  # :keys.subtokens    # => [:keys, :key, :ke, :k]
-  # :keys.subtokens(2) # => [:keys, :key, :ke]
-  #
-  def subtokens from_length = 1
-    sub = self.id2name
-    
-    size = sub.size
-    from_length = size + from_length if from_length < 0
-    from_length = size if size < from_length
-    
-    result = [self]
-    size.downto(from_length + 1) { result << sub.chop!.intern }
-    result
-  end
-  
-  # TODO Duplicate code.
+  # :keys.each_subtoken    # => yields each of [:keys, :key, :ke, :k]
+  # :keys.each_subtoken(2) # => yields each of [:keys, :key, :ke]
   #
   def each_subtoken from_length = 1
     sub = self.id2name

data/lib/picky/generator.rb

@@ -187,8 +187,6 @@ module Picky
         exclaim "#{entry} \x1b[31mexists\x1b[m, skipping."
       end
       
-      # TODO Remove?
-      #
       def exclaim something
         puts something
       end

data/lib/picky/index/bundle.rb

@@ -22,17 +22,19 @@ module Index
   class Bundle
     
     attr_reader   :identifier, :files
-    attr_accessor :index, :weights, :similarity, :similarity_strategy
+    attr_accessor :index, :weights, :similarity, :configuration, :similarity_strategy
     
-    delegate :[], :[]=, :clear, :to => :index
+    delegate :clear,    :to => :index
+    delegate :[], :[]=, :to => :configuration
     
     def initialize name, configuration, similarity_strategy
       @identifier = "#{configuration.identifier} (#{name})"
       @files      = Files.new name, configuration
       
-      @index      = {}
-      @weights    = {}
-      @similarity = {}
+      @index         = {}
+      @weights       = {}
+      @similarity    = {}
+      @configuration = {} # A hash with config options.
       
       @similarity_strategy = similarity_strategy
     end

data/lib/picky/index/file/basic.rb

@@ -33,15 +33,13 @@ module Index
       # the directory the index file is in.
       #
       def backup
-        prepare_backup backup_path
+        prepare_backup backup_directory
         FileUtils.cp cache_path, target, verbose: true
       end
       # The backup directory of this file.
       # Equal to the file's dirname plus /backup
       #
-      # TODO: Rename to backup_dir.
-      #
-      def backup_path
+      def backup_directory
         ::File.join ::File.dirname(cache_path), 'backup'
       end
       # Prepares the backup directory for the file.
@@ -58,8 +56,6 @@ module Index
       end
       # The backup filename.
       #
-      # TODO: Duplicate work done here?
-      #
       def backup_file_path_of path
         dir, name = ::File.split path
         ::File.join dir, 'backup', name

data/lib/picky/index/files.rb

@@ -3,22 +3,23 @@ module Index
   class Files
     
     attr_reader :bundle_name
-    attr_reader :prepared, :index, :similarity, :weights
+    attr_reader :prepared, :index, :weights, :similarity, :configuration
     
-    delegate :index_name, :category_name, :to => :@configuration
+    delegate :index_name, :category_name, :to => :@config
     
-    def initialize bundle_name, configuration
-      @bundle_name   = bundle_name
-      @configuration = configuration
+    def initialize bundle_name, config
+      @bundle_name = bundle_name
+      @config      = config
       
       # Note: We marshal the similarity, as the
       #       Yajl json lib cannot load symbolized
       #       values, just keys.
       #
-      @prepared   = File::Text.new    configuration.prepared_index_path
-      @index      = File::JSON.new    configuration.index_path(bundle_name, :index)
-      @similarity = File::Marshal.new configuration.index_path(bundle_name, :similarity)
-      @weights    = File::JSON.new    configuration.index_path(bundle_name, :weights)
+      @prepared      = File::Text.new    config.prepared_index_path
+      @index         = File::JSON.new    config.index_path(bundle_name, :index)
+      @weights       = File::JSON.new    config.index_path(bundle_name, :weights)
+      @similarity    = File::Marshal.new config.index_path(bundle_name, :similarity)
+      @configuration = File::JSON.new    config.index_path(bundle_name, :configuration)
     end
     
     # Delegators.
@@ -35,11 +36,14 @@ module Index
     def dump_index index_hash
       index.dump index_hash
     end
+    def dump_weights weights_hash
+      weights.dump weights_hash
+    end
     def dump_similarity similarity_hash
       similarity.dump similarity_hash
     end
-    def dump_weights weights_hash
-      weights.dump weights_hash
+    def dump_configuration configuration_hash
+      configuration.dump configuration_hash
     end
     
     # Loading.
@@ -53,6 +57,9 @@ module Index
     def load_weights
       weights.load
     end
+    def load_configuration
+      configuration.load
+    end
     
     # Cache ok?
     #
@@ -82,16 +89,18 @@ module Index
     #
     def backup
       index.backup
-      similarity.backup
       weights.backup
+      similarity.backup
+      configuration.backup
     end
     
     # Restores the indexes from the "backup" directory.
     #
     def restore
       index.restore
-      similarity.restore
       weights.restore
+      similarity.restore
+      configuration.restore
     end
     
     
@@ -99,8 +108,9 @@ module Index
     #
     def delete
       index.delete
-      similarity.delete
       weights.delete
+      similarity.delete
+      configuration.delete
     end
     
   end

data/lib/picky/indexed/bundle.rb

@@ -21,25 +21,13 @@ module Indexed
       @weights[sym]
     end
     
-    # Load the data from the db.
-    #
-    def load_from_index_file
-      load_from_index_generation_message
-      clear
-      retrieve
-    end
-    # Notifies the user that the index is being loaded.
-    #
-    def load_from_index_generation_message
-      timed_exclaim "LOAD INDEX #{identifier}."
-    end
-    
     # Loads all indexes.
     #
     def load
       load_index
-      load_similarity
       load_weights
+      load_similarity
+      load_configuration
     end
     # Loads the core index.
     #
@@ -56,6 +44,11 @@ module Indexed
     def load_similarity
       self.similarity = files.load_similarity
     end
+    # Loads the configuration.
+    #
+    def load_configuration
+      self.configuration = files.load_configuration
+    end
     
   end
 end

data/lib/picky/indexed/category.rb

@@ -7,7 +7,8 @@ module Indexed
   #
   class Category
     
-    attr_reader :exact, :partial, :identifier, :name
+    attr_accessor :exact, :partial
+    attr_reader   :identifier, :name
     
     #
     #

data/lib/picky/indexed/wrappers/bundle/calculation.rb

@@ -0,0 +1,35 @@
+module Indexed
+  module Wrappers
+    
+    module Bundle
+      
+      # A calculation rewrites the symbol into a float.
+      #
+      # TODO I really need to allow integers as keys. The code below is just not ok.
+      #
+      class Calculation < Wrapper
+        
+        #
+        #
+        def recalculate float
+          float
+        end
+        
+        #
+        #
+        def ids sym
+          @bundle.ids recalculate(sym.to_s.to_f).to_s.to_sym
+        end
+        
+        #
+        #
+        def weight sym
+          @bundle.weight recalculate(sym.to_s.to_f).to_s.to_sym
+        end
+        
+      end
+      
+    end
+    
+  end
+end

data/lib/picky/indexed/wrappers/bundle/location.rb

@@ -0,0 +1,40 @@
+module Indexed
+  module Wrappers
+    
+    module Bundle
+      
+      # A location calculation recalculates a location to the Picky internal location.
+      #
+      class Location < Calculation
+        
+        def initialize bundle, options = {}
+          super bundle
+          
+          precision  = options[:precision] || 1
+          user_grid  = options[:grid] || raise("Gridsize needs to be given for location #{bundle.identifier}.")
+          
+          @calculation         = Calculations::Location.new user_grid, precision
+        end
+        
+        #
+        #
+        def recalculate float
+          @calculation.recalculate float
+        end
+        
+        #
+        #
+        def load
+          # Load first the bundle, then extract the config.
+          #
+          bundle.load
+          minimum = bundle[:location_minimum] || raise("Configuration :location_minimum for #{bundle.identifier} missing.")
+          @calculation.minimum = minimum
+        end
+        
+      end
+      
+    end
+    
+  end
+end

data/lib/picky/indexed/wrappers/bundle/wrapper.rb

@@ -0,0 +1,25 @@
+module Indexed
+  module Wrappers
+    
+    # Per Bundle wrappers.
+    #
+    module Bundle
+      
+      # Base wrapper. Just delegates all methods to the bundle.
+      #
+      class Wrapper
+        
+        attr_reader :bundle
+        
+        def initialize bundle
+          @bundle = bundle
+        end
+        
+        delegate :load, :ids, :weight, :to => :@bundle
+        
+      end
+      
+    end
+    
+  end
+end

data/lib/picky/indexers/serial.rb

@@ -57,7 +57,7 @@ module Indexers
       end
     end
     def indexing_message
-      timed_exclaim "INDEX #{@configuration.identifier}"
+      timed_exclaim "INDEX #{@configuration.identifier}" # TODO from ...
     end
     
   end

data/lib/picky/indexing/bundle.rb

@@ -120,6 +120,7 @@ module Indexing
       dump_index
       dump_similarity
       dump_weights
+      dump_configuration
     end
     # Dumps the core index.
     #
@@ -139,6 +140,12 @@ module Indexing
       timed_exclaim "DUMP SIMILARITY #{identifier}."
       files.dump_similarity similarity
     end
+    # Dumps the similarity index.
+    #
+    def dump_configuration
+      timed_exclaim "DUMP CONFIGURATION #{identifier}."
+      files.dump_configuration configuration
+    end
     
     # Alerts the user if an index is missing.
     #

data/lib/picky/indexing/category.rb

@@ -4,15 +4,32 @@ module Indexing
     
     attr_reader :exact, :partial, :name, :configuration, :indexer
     
+    # Mandatory params:
+    #  * name: Category name to use as identifier and file names.
+    #  * index: Index to which this category is attached to.
+    # Options:
+    #  * partial: Partial::None.new, Partial::Substring.new(from:start_char, to:up_to_char) (defaults from:-3, to:-1)
+    #  * similarity: Similarity::None.new (default), Similarity::Phonetic.new(amount_of_similarly_linked_words)
+    #  * source: Use if the category should use a different source.
+    #  * from: The source category identifier to take the data from.
+    #
+    # Advanced Options (TODO):
+    #
+    #  * weights: 
+    #  * tokenizer: 
+    #  * exact_indexing_bundle: 
+    #  * partial_indexing_bundle: 
+    #
     def initialize name, index, options = {}
       @name = name
+      @from = options[:from]
       
       # Now we have enough info to combine the index and the category.
       #
       @configuration = Configuration::Index.new index, self
       
       @tokenizer = options[:tokenizer] || Tokenizers::Index.default
-      @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer #, :as => options[:as] # TODO option as.
+      @indexer = Indexers::Serial.new configuration, options[:source], @tokenizer
       
       # TODO Push into Bundle.
       #
@@ -27,6 +44,10 @@ module Indexing
     delegate :identifier, :prepare_index_directory, :to => :configuration
     delegate :source, :source=, :tokenizer, :tokenizer=, :to => :indexer
     
+    def from
+      @from || name
+    end
+    
     # TODO Spec.
     #
     def backup_caches

data/lib/picky/loader.rb

@@ -108,6 +108,10 @@ module Loader
     #
     load_relative 'character_substituters/west_european'
     
+    # Calculations.
+    #
+    load_relative 'calculations/location'
+    
     # Signal handling
     #
     load_relative 'signals'
@@ -188,6 +192,12 @@ module Loader
     
     load_relative 'indexed/wrappers/exact_first'
     
+    # Bundle Wrapper
+    #
+    load_relative 'indexed/wrappers/bundle/wrapper'
+    load_relative 'indexed/wrappers/bundle/calculation'
+    load_relative 'indexed/wrappers/bundle/location'
+    
     # Tokens.
     #
     load_relative 'query/token'

data/lib/picky/sources/couch.rb

@@ -24,7 +24,7 @@ module Sources
     # Harvests the data to index.
     #
     def harvest type, category
-      category_name = category.name.to_s
+      category_name = category.from.to_s
       get_data do |doc|
         yield doc['_id'].to_i, doc[category_name] || next
       end

data/lib/picky/sources/csv.rb

@@ -7,12 +7,14 @@ module Sources
   
   class CSV < Base
     
-    attr_reader :file_name, :category_names
+    attr_reader :file_name, :csv_options, :category_names
     
     def initialize *category_names, options
       require 'csv'
       @category_names = category_names
-      @file_name   = Hash === options && options[:file] || raise_no_file_given(category_names)
+      
+      @csv_options    = Hash === options && options || {}
+      @file_name      = @csv_options.delete(:file) || raise_no_file_given(category_names)
     end
     
     #
@@ -24,7 +26,7 @@ module Sources
     # Harvests the data to index.
     #
     def harvest _, category
-      index = category_names.index category.name
+      index = category_names.index category.from
       get_data do |ary|
         indexed_id = ary.shift.to_i # TODO is to_i necessary?
         text       = ary[index]
@@ -37,7 +39,7 @@ module Sources
     #
     #
     def get_data &block
-      ::CSV.foreach file_name, &block
+      ::CSV.foreach file_name, csv_options, &block
     end
     
   end

data/lib/picky/sources/db.rb

@@ -126,7 +126,7 @@ module Sources
     # Base harvest statement for dbs.
     #
     def harvest_statement type, category
-      "SELECT indexed_id, #{category.name} FROM #{snapshot_table_name(type)} st"
+      "SELECT indexed_id, #{category.from} FROM #{snapshot_table_name(type)} st"
     end
     
     # Override in subclasses.

data/lib/picky/sources/delicious.rb

@@ -19,7 +19,7 @@ module Sources
     def harvest _, category
       get_data do |uid, data|
         indexed_id = uid
-        text = data[category.name]
+        text = data[category.from]
         next unless text
         text.force_encoding 'utf-8' # TODO Still needed?
         yield indexed_id, text

data/lib/picky/sources/wrappers/base.rb

@@ -4,12 +4,13 @@ module Sources
     
     class Base
       
-      attr_reader :backend
+      attr_reader :backend, :category
       
-      # Wraps a backend
+      # Wraps an indexing category.
       #
-      def initialize backend
-        @backend = backend
+      def initialize category
+        @category = category
+        @backend  = category.source
       end
       
       # Default is delegation for all methods