picky 0.11.2 → 0.12.0

data/lib/picky/Index_api.rb

@@ -0,0 +1,49 @@
+# This class defines the indexing and index API.
+#
+# Note: An Index holds both an *Indexed*::*Index* and an *Indexing*::*Type*.
+#
+class IndexAPI
+  
+  # TODO Delegation.
+  #
+  
+  attr_reader :name, :indexing, :indexed
+  
+  def initialize name, source, options = {}
+    @name     = name
+    @indexing = Indexing::Index.new name, source, options
+    @indexed  = Indexed::Index.new  name, options
+    
+    # Centralized registry.
+    #
+    Indexes.register self
+  end
+  
+  # API.
+  #
+  # TODO Spec! Doc!
+  #
+  def define_category category_name, options = {}
+    category_name = category_name.to_sym
+    
+    indexing_category = indexing.add_category category_name, options
+    indexed_category  = indexed.add_category  category_name, options
+    
+    yield indexing_category, indexed_category if block_given?
+    
+    self
+  end
+  alias category define_category
+  
+  def define_location name, options = {}
+    grid      = options[:grid]
+    precision = options[:precision]
+    
+    define_category name, options do |indexing, _|
+      indexing.source    = Sources::Wrappers::Location.new indexing.source, grid: grid, precision: precision
+      indexing.tokenizer = Tokenizers::Index.new
+    end
+  end
+  alias location define_location
+  
+end

data/lib/picky/alias_instances.rb

@@ -1 +1,4 @@
-Indexes = Types.new
+# This provides a nice accessor for users
+# who want to use the index API.
+#
+Indexes = IndexesAPI.new

data/lib/picky/application.rb

@@ -4,24 +4,37 @@ class Application
   
   class << self
     
+    # API
+    #
+    
     # Returns a configured tokenizer that
     # is used for indexing by default.
     # 
     def default_indexing options = {}
-      indexing.default_tokenizer options
+      Tokenizers::Index.default = Tokenizers::Index.new(options)
     end
     
     # Returns a configured tokenizer that
     # is used for querying by default.
     # 
     def default_querying options = {}
-      querying.default_tokenizer options
+      Tokenizers::Query.default = Tokenizers::Query.new(options)
+    end
+    
+    # Returns a new index frontend for configuring the index.
+    #
+    def index *args
+      IndexAPI.new *args
     end
     
     # Routes.
     #
     delegate :route, :root, :to => :routing
     
+    #
+    # API
+    
+    
     # An application simply delegates to the routing to handle a request.
     #
     def call env
@@ -30,18 +43,6 @@ class Application
     def routing
       @routing ||= Routing.new
     end
-    def indexing
-      @indexing ||= Configuration::Indexes.new
-    end
-    def querying
-      @queries ||= Configuration::Queries.new
-    end
-    
-    # "API".
-    #
-    def index *args
-      ::Type.new *args
-    end
     
     # Finalize the subclass as soon as it
     # has finished loading.
@@ -64,7 +65,7 @@ class Application
       routing.freeze
     end
     
-    # TODO Add more info.
+    # TODO Add more info if possible.
     #
     def to_s
       "#{self.name}:\n#{routing}"

data/lib/picky/cacher/partial/{subtoken.rb → substring.rb}

@@ -2,7 +2,7 @@ module Cacher
 
   module Partial
     
-    # Generates the right subtokens for use in the subtoken strategy.
+    # Generates the right substrings for use in the substring strategy.
     #
     class SubstringGenerator
       
@@ -27,38 +27,39 @@ module Cacher
     
     # The subtoken partial strategy.
     #
-    # If given
-    # "florian"
-    # will index
-    # "floria"
-    # "flori"
-    # "flor"
-    # "flo"
-    # "fl"
-    # "f"
-    # Depending on what the given from value is. (Example with from == 1)
+    # If given "florian"
+    # it will index "floria", "flori", "flor", "flo", "fl", "f"
+    # (Depending on what the given from value is, the example is with option from: 1)
     #
     class Substring < Strategy
       
-      # Down to is how far it will go down in generating the subtokens.
+      # The from option signifies where in the symbol it
+      # will start in generating the subtokens.
       #
       # Examples:
-      # With :hello, and to -1
-      # * down to == 1: [:hello, :hell, :hel, :he, :h]
-      # * down to == 4: [:hello, :hell]
       #
-      # With :hello, and to -2
-      # * down to == 1: [:hell, :hel, :he, :h]
-      # * down to == 4: [:hell]
+      # With :hello, and to: -1 (default)
+      # * from: 1 # => [:hello, :hell, :hel, :he, :h]
+      # * from: 4 # => [:hello, :hell]
+      #
+      # With :hello, and to: -2
+      # * from: 1 # => [:hell, :hel, :he, :h]
+      # * from: 4 # => [:hell]
       #
       def initialize options = {}
         from     = options[:from] || 1
         to = options[:to] || -1
         @generator = SubstringGenerator.new from, to
       end
+      
+      # Delegator to generator#from.
+      #
       def from
         @generator.from
       end
+      
+      # Delegator to generator#to.
+      #
       def to
         @generator.to
       end

data/lib/picky/{character_substitution/european.rb → character_substituters/west_european.rb}

@@ -1,11 +1,11 @@
 # encoding: utf-8
 #
-module CharacterSubstitution
+module CharacterSubstituters
   # Substitutes Umlauts like
   # ä, ö, ü => ae, oe, ue.
   # (and more, see specs)
   #
-  class European
+  class WestEuropean
     
     def initialize
       @chars = ActiveSupport::Multibyte.proxy_class

data/lib/picky/configuration/index.rb

@@ -0,0 +1,67 @@
+module Configuration
+  
+  # Holds the configuration for a
+  # index/category combination.
+  #
+  # TODO Rename paths?
+  #
+  class Index
+    
+    attr_reader :index, :category
+    
+    def initialize index, category
+      @index    = index
+      @category = category
+    end
+    
+    def index_name
+      @index_name ||= index.name
+    end
+    def category_name
+      @category_name ||= category.name
+    end
+    
+    #
+    #
+    def index_path bundle_name, name
+      "#{index_directory}/#{category_name}_#{bundle_name}_#{name}"
+    end
+    
+    # Was: search_index_file_name
+    #
+    def prepared_index_path
+      @prepared_index_path ||= "#{index_directory}/prepared_#{category_name}_index"
+    end
+    def prepared_index_file &block
+      @prepared_index_file ||= ::Index::File::Text.new prepared_index_path
+      @prepared_index_file.open_for_indexing &block
+    end
+    
+    # def file_name
+    #   @file_name ||= "#{@index_name}_#{@category_name}"
+    # end
+    
+    def identifier
+      @identifier ||= "#{index_name} #{category_name}"
+    end
+    
+    def self.index_root
+      @index_root ||= "#{PICKY_ROOT}/index"
+    end
+    def index_root
+      self.class.index_root
+    end
+    # Was: cache_directory
+    #
+    def index_directory
+      @index_directory ||= "#{index_root}/#{PICKY_ENVIRONMENT}/#{index_name}"
+    end
+    # Was: prepare_cache_directory
+    #
+    def prepare_index_directory
+      FileUtils.mkdir_p index_directory
+    end
+    
+  end
+  
+end

data/lib/picky/cores.rb

@@ -15,6 +15,9 @@ class Cores
   #  * max: Maximum # of processors to use. Default is all it can get.
   #
   def self.forked ary_or_generator, options = {}
+    return if ary_or_generator.empty?
+    raise "Block argument needed when running Cores.forked" unless block_given?
+    
     ary_or_generator = ary_or_generator.sort_by { rand } if options[:randomly]
     generator        = ary_or_generator.each
     

data/lib/picky/index/bundle.rb

@@ -1,63 +1,47 @@
-# encoding: utf-8
-#
 module Index
-
-  # This is the ACTUAL index.
+  # A Bundle is a number of indexes
+  # per [index, category] combination.
   #
-  # Handles exact index, partial index, weights index, and similarity index.
+  # At most, there are three indexes:
+  # * *core* index (always used)
+  # * *weights* index (always used)
+  # * *similarity* index (used with similarity)
+  # 
+  # In Picky, indexing is separated from the index
+  # handling itself through a parallel structure.
   #
-  # Delegates file handling and checking to a Index::Files object.
+  # Both use methods provided by this base class, but
+  # have very different goals:
   #
-  class Bundle < ::Bundle
+  # * *Indexing*::*Bundle* is just concerned with creating index files
+  #   and providing helper functions to e.g. check the indexes.
+  #
+  # * *Index*::*Bundle* is concerned with loading these index files into
+  #   memory and looking up search data as fast as possible.
+  #
+  class Bundle
     
-    # Get the ids for the text.
-    #
-    def ids text
-      @index[text] || []
-    end
-    # Get a weight for the text.
-    #
-    def weight text
-      @weights[text]
-    end
+    attr_reader   :identifier, :files
+    attr_accessor :index, :weights, :similarity, :similarity_strategy
     
-    # Load the data from the db.
-    #
-    def load_from_index_file
-      load_from_index_generation_message
-      clear
-      retrieve
-    end
-    def load_from_index_generation_message
-      timed_exclaim "LOAD INDEX #{identifier}."
-    end
-    # Retrieves the data into the index.
-    #
-    def retrieve
-      files.retrieve do |id, token|
-        initialize_index_for token
-        index[token] << id
-      end
-    end
-    def initialize_index_for token
-      index[token] ||= []
+    delegate :[], :[]=, :clear, :to => :index
+    
+    def initialize name, configuration, similarity_strategy
+      @identifier = "#{configuration.identifier} (#{name})"
+      @files      = Files.new name, configuration
+      
+      @index      = {}
+      @weights    = {}
+      @similarity = {}
+      
+      @similarity_strategy = similarity_strategy
     end
     
-    # Loads all indexes into this category.
+    # Get a list of similar texts.
     #
-    def load
-      load_index
-      load_similarity
-      load_weights
-    end
-    def load_index
-      self.index = files.load_index
-    end
-    def load_similarity
-      self.similarity = files.load_similarity
-    end
-    def load_weights
-      self.weights = files.load_weights
+    def similar text
+      code = similarity_strategy.encoded text
+      code && @similarity[code] || []
     end
     
   end

data/lib/picky/index/file/basic.rb

@@ -1,43 +1,71 @@
 module Index
   
+  # Handles all aspects of index files, such as dumping/loading.
+  #
   module File
     
+    # Base class for all index files.
+    #
+    # Provides necessary helper methods for its
+    # subclasses.
+    # Not directly useable, as it does not provide
+    # dump/load methods.
+    #
     class Basic
       
       attr_reader :cache_path
       
+      # An index cache takes a path, without file extension,
+      # which will be provided by the subclasses.
+      #
       def initialize cache_path
         @cache_path = "#{cache_path}.#{extension}"
       end
       
+      # The default extension for index files is "index".
+      #
       def extension
         :index
       end
       
-      # Backup.
+      # Will copy the index file to a location that
+      # is in a directory named "backup" right under
+      # the directory the index file is in.
       #
       def backup
         prepare_backup backup_path
         FileUtils.cp cache_path, target, verbose: true
       end
+      # The backup directory of this file.
+      # Equal to the file's dirname plus /backup
+      #
+      # TODO: Rename to backup_dir.
+      #
       def backup_path
         ::File.join ::File.dirname(cache_path), 'backup'
       end
+      # Prepares the backup directory for the file.
+      #
       def prepare_backup target
         FileUtils.mkdir target unless Dir.exists?(target)
       end
       
-      # Restore.
+      # Copies the file from its backup location back
+      # to the original location.
       #
       def restore
         FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
       end
+      # The backup filename.
+      #
+      # TODO: Duplicate work done here?
+      #
       def backup_file_path_of path
         dir, name = ::File.split path
         ::File.join dir, 'backup', name
       end
       
-      # Delete.
+      # Deletes the file.
       #
       def delete
         `rm -Rf #{cache_path}`
@@ -46,16 +74,22 @@ module Index
       # Checks.
       #
       
-      # Is the cache small?
+      # Is this cache file suspiciously small?
+      # (less than 8 Bytes of size)
       #
       def cache_small?
         size_of(cache_path) < 8
       end
-      # Is the cache ok? I.e. larger than four Bytes in size.
+      # Is the cache ok? (existing and larger than
+      # zero Bytes in size)
+      #
+      # A small cache is still ok.
       #
       def cache_ok?
         size_of(cache_path) > 0
       end
+      # Extracts the size of the file in Bytes.
+      #
       def size_of path
         `ls -l #{path} | awk '{print $5}'`.to_i
       end

data/lib/picky/index/file/json.rb

@@ -2,17 +2,27 @@ module Index
   
   module File
     
+    # Index files dumped in the JSON format.
+    #
     class JSON < Basic
       
+      # Uses the extension "json".
+      #
       def extension
         :json
       end
+      # Loads the index hash from json format.
+      #
       def load
         Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true
       end
+      # Dumps the index hash in json format.
+      #
       def dump hash
         hash.dump_json cache_path
       end
+      # A json file does not provide retrieve functionality.
+      #
       def retrieve
         raise "Can't retrieve from JSON file. Use text file."
       end

data/lib/picky/index/file/marshal.rb

@@ -2,17 +2,27 @@ module Index
   
   module File
     
+    # Index data in the Ruby Marshal format.
+    #
     class Marshal < Basic
       
+      # Uses the extension "dump".
+      #
       def extension
         :dump
       end
+      # Loads the index hash from marshal format.
+      #
       def load
         ::Marshal.load ::File.open(cache_path, 'r:binary')
       end
+      # Dumps the index hash in marshal format.
+      #
       def dump hash
         hash.dump_marshalled cache_path
       end
+      # A marshal file does not provide retrieve functionality.
+      #
       def retrieve
         raise "Can't retrieve from marshalled file. Use text file."
       end

data/lib/picky/index/file/text.rb

@@ -2,18 +2,33 @@ module Index
   
   module File
     
+    # Index data dumped in the text format.
+    #
     class Text < Basic
       
+      # Uses the extension "txt".
+      #
       def extension
         :txt
       end
+      # Text files are used exclusively for
+      # prepared data files.
+      #
       def load
         raise "Can't load from text file. Use JSON or Marshal."
       end
+      # Text files are used exclusively for
+      # prepared data files. 
+      #
       def dump hash
         raise "Can't dump to text file. Use JSON or Marshal."
       end
       
+      # Retrieves prepared index data in the form
+      # * id,data\n
+      # * id,data\n
+      # * id,data\n
+      #
       # Yields an id and a symbol token.
       #
       def retrieve
@@ -26,6 +41,13 @@ module Index
         end
       end
       
+      #
+      #
+      def open_for_indexing &block
+        ::File.open cache_path, 'w:binary', &block
+      end
+      
+      
     end
     
   end

data/lib/picky/index/files.rb

@@ -2,59 +2,34 @@ module Index
   
   class Files
     
-    attr_reader :bundle_name, :category_name, :type_name
+    attr_reader :bundle_name
     attr_reader :prepared, :index, :similarity, :weights
     
-    def initialize bundle_name, category_name, type_name
+    delegate :index_name, :category_name, :to => :@configuration
+    
+    def initialize bundle_name, configuration
       @bundle_name   = bundle_name
-      @category_name = category_name
-      @type_name     = type_name
+      @configuration = configuration
       
       # Note: We marshal the similarity, as the
       #       Yajl json lib cannot load symbolized
       #       values, just keys.
       #
-      @prepared   = File::Text.new    "#{cache_directory}/prepared_#{category_name}_index"
-      @index      = File::JSON.new    cache_path(:index)
-      @similarity = File::Marshal.new cache_path(:similarity)
-      @weights    = File::JSON.new    cache_path(:weights)
-    end
-    
-    # Paths.
-    #
-    
-    # Cache path, for File-s.
-    #
-    def cache_path name
-      ::File.join cache_directory, "#{bundle_name}_#{category_name}_#{name}"
+      @prepared   = File::Text.new    configuration.prepared_index_path
+      @index      = File::JSON.new    configuration.index_path(bundle_name, :index)
+      @similarity = File::Marshal.new configuration.index_path(bundle_name, :similarity)
+      @weights    = File::JSON.new    configuration.index_path(bundle_name, :weights)
     end
     
-    # Point to category.
+    # Delegators.
     #
-    def search_index_root
-      ::File.join PICKY_ROOT, 'index'
-    end
     
-    # Create directory and parent directories.
+    # Retrieving data.
     #
-    def create_directory
-      FileUtils.mkdir_p cache_directory
-    end
-    # TODO Move to config. Duplicate Code in field.rb.
-    #
-    def cache_directory
-      "#{search_index_root}/#{PICKY_ENVIRONMENT}/#{type_name}"
-    end
     def retrieve &block
       prepared.retrieve &block
     end
     
-    # Single index/similarity/weights files delegation.
-    #
-    
-    # Delegators.
-    #
-    
     # Dumping.
     #
     def dump_index index_hash