picky 1.4.1 → 1.4.2

data/lib/picky/index/file/basic.rb

@@ -1,97 +0,0 @@
-module Index
-  
-  # Handles all aspects of index files, such as dumping/loading.
-  #
-  module File
-    
-    # Base class for all index files.
-    #
-    # Provides necessary helper methods for its
-    # subclasses.
-    # Not directly useable, as it does not provide
-    # dump/load methods.
-    #
-    class Basic
-      
-      attr_reader :cache_path
-      
-      # An index cache takes a path, without file extension,
-      # which will be provided by the subclasses.
-      #
-      def initialize cache_path
-        @cache_path = "#{cache_path}.#{extension}"
-      end
-      
-      # The default extension for index files is "index".
-      #
-      def extension
-        :index
-      end
-      
-      # Will copy the index file to a location that
-      # is in a directory named "backup" right under
-      # the directory the index file is in.
-      #
-      def backup
-        prepare_backup backup_directory
-        FileUtils.cp cache_path, target, verbose: true
-      end
-      # The backup directory of this file.
-      # Equal to the file's dirname plus /backup
-      #
-      def backup_directory
-        ::File.join ::File.dirname(cache_path), 'backup'
-      end
-      # Prepares the backup directory for the file.
-      #
-      def prepare_backup target
-        FileUtils.mkdir target unless Dir.exists?(target)
-      end
-      
-      # Copies the file from its backup location back
-      # to the original location.
-      #
-      def restore
-        FileUtils.cp backup_file_path_of(cache_path), cache_path, verbose: true
-      end
-      # The backup filename.
-      #
-      def backup_file_path_of path
-        dir, name = ::File.split path
-        ::File.join dir, 'backup', name
-      end
-      
-      # Deletes the file.
-      #
-      def delete
-        `rm -Rf #{cache_path}`
-      end
-      
-      # Checks.
-      #
-      
-      # Is this cache file suspiciously small?
-      # (less than 8 Bytes of size)
-      #
-      def cache_small?
-        size_of(cache_path) < 8
-      end
-      # Is the cache ok? (existing and larger than
-      # zero Bytes in size)
-      #
-      # A small cache is still ok.
-      #
-      def cache_ok?
-        size_of(cache_path) > 0
-      end
-      # Extracts the size of the file in Bytes.
-      #
-      def size_of path
-        `ls -l #{path} | awk '{print $5}'`.to_i
-      end
-      
-    end
-    
-  end
-  
-end

data/lib/picky/index/file/json.rb

@@ -1,34 +0,0 @@
-module Index
-  
-  module File
-    
-    # Index files dumped in the JSON format.
-    #
-    class JSON < Basic
-      
-      # Uses the extension "json".
-      #
-      def extension
-        :json
-      end
-      # Loads the index hash from json format.
-      #
-      def load
-        Yajl::Parser.parse ::File.open(cache_path, 'r'), symbolize_keys: true
-      end
-      # Dumps the index hash in json format.
-      #
-      def dump hash
-        hash.dump_json cache_path
-      end
-      # A json file does not provide retrieve functionality.
-      #
-      def retrieve
-        raise "Can't retrieve from JSON file. Use text file."
-      end
-      
-    end
-    
-  end
-  
-end

data/lib/picky/index/file/marshal.rb

@@ -1,34 +0,0 @@
-module Index
-  
-  module File
-    
-    # Index data in the Ruby Marshal format.
-    #
-    class Marshal < Basic
-      
-      # Uses the extension "dump".
-      #
-      def extension
-        :dump
-      end
-      # Loads the index hash from marshal format.
-      #
-      def load
-        ::Marshal.load ::File.open(cache_path, 'r:binary')
-      end
-      # Dumps the index hash in marshal format.
-      #
-      def dump hash
-        hash.dump_marshalled cache_path
-      end
-      # A marshal file does not provide retrieve functionality.
-      #
-      def retrieve
-        raise "Can't retrieve from marshalled file. Use text file."
-      end
-      
-    end
-    
-  end
-  
-end

data/lib/picky/index/file/text.rb

@@ -1,56 +0,0 @@
-module Index
-  
-  module File
-    
-    # Index data dumped in the text format.
-    #
-    class Text < Basic
-      
-      # Uses the extension "txt".
-      #
-      def extension
-        :txt
-      end
-      # Text files are used exclusively for
-      # prepared data files.
-      #
-      def load
-        raise "Can't load from text file. Use JSON or Marshal."
-      end
-      # Text files are used exclusively for
-      # prepared data files. 
-      #
-      def dump hash
-        raise "Can't dump to text file. Use JSON or Marshal."
-      end
-      
-      # Retrieves prepared index data in the form
-      # * id,data\n
-      # * id,data\n
-      # * id,data\n
-      #
-      # Yields an id string and a symbol token.
-      #
-      def retrieve
-        id    = nil
-        token = nil
-        ::File.open(cache_path, 'r:binary') do |file|
-          file.each_line do |line|
-            id, token = line.split ?,, 2
-            yield id, (token.chomp! || token).to_sym
-          end
-        end
-      end
-      
-      #
-      #
-      def open_for_indexing &block
-        ::File.open cache_path, 'w:binary', &block
-      end
-      
-      
-    end
-    
-  end
-  
-end

data/lib/picky/index/files.rb

@@ -1,118 +0,0 @@
-module Index
-  
-  class Files
-    
-    attr_reader :bundle_name
-    attr_reader :prepared, :index, :weights, :similarity, :configuration
-    
-    delegate :index_name, :category_name, :to => :@config
-    
-    def initialize bundle_name, config
-      @bundle_name = bundle_name
-      @config      = config
-      
-      # Note: We marshal the similarity, as the
-      #       Yajl json lib cannot load symbolized
-      #       values, just keys.
-      #
-      @prepared      = File::Text.new    config.prepared_index_path
-      @index         = File::JSON.new    config.index_path(bundle_name, :index)
-      @weights       = File::JSON.new    config.index_path(bundle_name, :weights)
-      @similarity    = File::Marshal.new config.index_path(bundle_name, :similarity)
-      @configuration = File::JSON.new    config.index_path(bundle_name, :configuration)
-    end
-    
-    # Delegators.
-    #
-    
-    # Retrieving data.
-    #
-    def retrieve &block
-      prepared.retrieve &block
-    end
-    
-    # Dumping.
-    #
-    def dump_index index_hash
-      index.dump index_hash
-    end
-    def dump_weights weights_hash
-      weights.dump weights_hash
-    end
-    def dump_similarity similarity_hash
-      similarity.dump similarity_hash
-    end
-    def dump_configuration configuration_hash
-      configuration.dump configuration_hash
-    end
-    
-    # Loading.
-    #
-    def load_index
-      index.load
-    end
-    def load_similarity
-      similarity.load
-    end
-    def load_weights
-      weights.load
-    end
-    def load_configuration
-      configuration.load
-    end
-    
-    # Cache ok?
-    #
-    def index_cache_ok?
-      index.cache_ok?
-    end
-    def similarity_cache_ok?
-      similarity.cache_ok?
-    end
-    def weights_cache_ok?
-      weights.cache_ok?
-    end
-    
-    # Cache small?
-    #
-    def index_cache_small?
-      index.cache_small?
-    end
-    def similarity_cache_small?
-      similarity.cache_small?
-    end
-    def weights_cache_small?
-      weights.cache_small?
-    end
-    
-    # Copies the indexes to the "backup" directory.
-    #
-    def backup
-      index.backup
-      weights.backup
-      similarity.backup
-      configuration.backup
-    end
-    
-    # Restores the indexes from the "backup" directory.
-    #
-    def restore
-      index.restore
-      weights.restore
-      similarity.restore
-      configuration.restore
-    end
-    
-    
-    # Delete all index files.
-    #
-    def delete
-      index.delete
-      weights.delete
-      similarity.delete
-      configuration.delete
-    end
-    
-  end
-  
-end

data/lib/picky/index_api.rb

@@ -1,175 +0,0 @@
-# This class defines the indexing and index API that is exposed to the user
-# as the #index method inside the Application class.
-#
-# It provides a single front for both indexing and index options. We suggest to always use the index API.
-#
-# Note: An Index holds both an *Indexed*::*Index* and an *Indexing*::*Type*.
-#
-class IndexAPI
-  
-  attr_reader :name, :indexing, :indexed
-  
-  # Create a new index with a given source.
-  #
-  # === Parameters
-  # * name: A name that will be used for the index directory and in the Picky front end.
-  # * source: Where the data comes from, e.g. Sources::CSV.new(...)
-  #
-  # === Options
-  # * result_identifier: Use if you'd like a different identifier/name in the results than the name of the index.
-  # * after_indexing: As of this writing only used in the db source. Executes the given after_indexing as SQL after the indexing process.
-  #
-  def initialize name, source, options = {}
-    @name     = name
-    @indexing = Indexing::Index.new name, source, options
-    @indexed  = Indexed::Index.new  name, options
-    
-    # Centralized registry.
-    #
-    Indexes.register self
-  end
-  
-  # Defines a searchable category on the index.
-  #
-  # === Parameters
-  # * category_name: This identifier is used in the front end, but also to categorize query text. For example, “title:hobbit” will narrow the hobbit query on categories with the identifier :title.
-  #
-  # === Options
-  # * partial: Partial::None.new or Partial::Substring.new(from: starting_char, to: ending_char). Default is Partial::Substring.new(from: -3, to: -1).
-  # * similarity: Similarity::None.new or Similarity::Phonetic.new(similar_words_searched). Default is Similarity::None.new.
-  # * qualifiers: An array of qualifiers with which you can define which category you’d like to search, for example “title:hobbit” will search for hobbit in just title categories. Example: qualifiers: [:t, :titre, :title] (use it for example with multiple languages). Default is the name of the category.
-  # * qualifier: Convenience options if you just need a single qualifier, see above. Example: qualifiers => :title. Default is the name of the category.
-  # * source: Use a different source than the index uses. If you think you need that, there might be a better solution to your problem. Please post to the mailing list first with your application.rb :)
-  # * from: Take the data from the data category with this name. Example: You have a source Sources::CSV.new(:title, file:'some_file.csv') but you want the category to be called differently. The you use from: define_category(:similar_title, :from => :title).
-  #
-  def define_category category_name, options = {}
-    category_name = category_name.to_sym
-    
-    indexing_category = indexing.define_category category_name, options
-    indexed_category  = indexed.define_category  category_name, options
-    
-    yield indexing_category, indexed_category if block_given?
-    
-    self
-  end
-  alias category define_category
-  
-  #  HIGHLY EXPERIMENTAL Try if you feel "beta" ;)
-  #
-  # Make this category range searchable with a fixed range. If you need other ranges, define another category with a different range value.
-  #
-  # Example:
-  # You have data values inside 1..100, and you want to have Picky return
-  # not only the results for 47 if you search for 47, but also results for
-  # 45, 46, or 47.2, 48.9, in a range of 2 around 47, so (45..49).
-  #
-  # Then you use:
-  #  my_index.define_ranged_category :values_inside_1_100, 2
-  #
-  # Optionally, you give it a precision value to reduce the error margin
-  # around 47 (Picky is a bit liberal).
-  #  my_index.define_ranged_category :values_inside_1_100, 2, precision: 5
-  #
-  # This will force Picky to maximally be wrong 5% of the given range value
-  # (5% of 2 = 0.1) instead of the default 20% (20% of 2 = 0.4).
-  #
-  # We suggest not to use much more than 5 as a higher precision is more performance intensive for less and less precision gain.
-  #
-  # == Protip 1
-  #
-  # Create two ranged categories to make an area search:
-  #   index.define_ranged_category :x, 1
-  #   index.define_ranged_category :y, 1
-  #
-  # Search for it using for example:
-  #   x:133, y:120
-  #
-  # This will search this square area (* = 133, 120: The "search" point entered):
-  # 
-  #    132       134
-  #     |         |
-  #   --|---------|-- 121
-  #     |         |
-  #     |    *    |
-  #     |         |
-  #   --|---------|-- 119
-  #     |         | 
-  #
-  # Note: The area does not need to be square, but can be rectangular.
-  #
-  # == Protip 2
-  #
-  # Create three ranged categories to make a volume search.
-  #
-  # Or go crazy and use 4 ranged categories for a space/time search! ;)
-  #
-  # === Parameters
-  # * category_name: The category_name as used in #define_category.
-  # * range: The range (in the units of your data values) around the query point where we search for results.
-  #
-  #  -----|<- range  ->*------------|-----
-  #
-  # === Options
-  # * precision: Default is 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
-  # * ... all options of #define_category.
-  #
-  def define_ranged_category category_name, range, options = {}
-    precision = options[:precision]
-    
-    options = { partial: Partial::None.new }.merge options
-    
-    define_category category_name, options do |indexing, indexed|
-      indexing.source    = Sources::Wrappers::Location.new indexing, grid: range, precision: precision
-      indexing.tokenizer = Tokenizers::Index.new
-      
-      exact_bundle    = Indexed::Wrappers::Bundle::Location.new indexed.exact, grid: range, precision: precision
-      indexed.exact   = exact_bundle
-      indexed.partial = exact_bundle # A partial token also uses the exact index.
-    end
-  end
-  alias ranged_category define_ranged_category
-  
-  #  HIGHLY EXPERIMENTAL Not correctly working yet. Try it if you feel "beta".
-  #
-  # Also a range search see #define_ranged_category, but on the earth's surface.
-  #
-  # Parameters:
-  # * name: The name as used in #define_category.
-  # * radius: The distance (in km) around the query point which we search for results.
-  #
-  # Note: Picky uses a square, not a circle. We hope that's ok for most usages.
-  #
-  #  -----------------------------
-  #  |                           |
-  #  |                           |
-  #  |                           |
-  #  |                           |
-  #  |                           |
-  #  |             *<-  radius ->|
-  #  |                           |
-  #  |                           |
-  #  |                           |
-  #  |                           |
-  #  |                           |
-  #  -----------------------------
-  #
-  # Options
-  # * precision: Default 1 (20% error margin, very fast), up to 5 (5% error margin, slower) makes sense.
-  # * from: The data category to take the data for this category from.
-  #
-  # TODO Redo. Will have to write a wrapper that combines two categories that are indexed simultaneously.
-  #
-  def define_map_location name, radius, options = {} # :nodoc:
-    # The radius is given as if all the locations were on the equator.
-    #
-    # TODO Need to recalculate since not many locations are on the equator ;) This is just a prototype.
-    #
-    # This calculates km -> longitude (degrees).
-    #
-    # A degree on the equator is equal to ~111,319.9 meters.
-    # So a km on the equator is equal to 0.00898312 degrees.
-    #
-    define_ranged_category name, radius * 0.00898312, options
-  end
-  alias map_location define_map_location
-end

data/lib/picky/indexed/bundle.rb

@@ -1,54 +0,0 @@
-# encoding: utf-8
-#
-module Indexed # :nodoc:all
-
-  # This is the _actual_ index.
-  #
-  # Handles exact/partial index, weights index, and similarity index.
-  #
-  # Delegates file handling and checking to an *Indexed*::*Files* object.
-  #
-  class Bundle < Index::Bundle
-    
-    # Get the ids for the given symbol.
-    #
-    def ids sym
-      @index[sym] || []
-    end
-    # Get a weight for the given symbol.
-    #
-    def weight sym
-      @weights[sym]
-    end
-    
-    # Loads all indexes.
-    #
-    def load
-      load_index
-      load_weights
-      load_similarity
-      load_configuration
-    end
-    # Loads the core index.
-    #
-    def load_index
-      self.index = files.load_index
-    end
-    # Loads the weights index.
-    #
-    def load_weights
-      self.weights = files.load_weights
-    end
-    # Loads the similarity index.
-    #
-    def load_similarity
-      self.similarity = files.load_similarity
-    end
-    # Loads the configuration.
-    #
-    def load_configuration
-      self.configuration = files.load_configuration
-    end
-    
-  end
-end

data/lib/picky/indexed/categories.rb

@@ -1,131 +0,0 @@
-module Indexed
-  
-  class Categories
-    
-    attr_reader :categories, :category_hash, :ignore_unassigned_tokens
-    
-    each_delegate :load_from_cache,
-                  :to => :categories
-    
-    # A list of indexed categories.
-    #
-    # Options:
-    #  * ignore_unassigned_tokens: Ignore the given token if it cannot be matched to a category.
-    #                              The default behaviour is that if a token does not match to
-    #                              any category, the query will not return anything (since a
-    #                              single token cannot be matched). If you set this option to
-    #                              true, any token that cannot be matched to a category will be
-    #                              simply ignored.
-    #                              Use this if only a few matched words are important, like for
-    #                              example of the query "Jonathan Myers 86455 Las Cucarachas"
-    #                              you only want to match the zipcode, to have the search engine
-    #                              display advertisements on the side for the zipcode.
-    #                              Nifty! :)
-    #
-    def initialize options = {}
-      clear
-      
-      @ignore_unassigned_tokens = options[:ignore_unassigned_tokens] || false
-    end
-    
-    # Clears both the array of categories and the hash of categories.
-    #
-    def clear
-      @categories    = []
-      @category_hash = {}
-    end
-    
-    # Add the given category to the list of categories.
-    #
-    def << category
-      categories << category
-      # Note: [category] is an optimization, since I need an array
-      #       of categories.
-      #       It's faster to just package it in an array on loading
-      #       Picky than doing it over and over with each query.
-      #
-      category_hash[category.name] = [category]
-    end
-    
-    # Return all possible combinations for the given token.
-    #
-    # This checks if it needs to also search through similar
-    # tokens, if for example, the token is one with ~.
-    # If yes, it puts together all solutions.
-    #
-    def possible_combinations_for token
-      token.similar? ? similar_possible_for(token) : possible_for(token)
-    end
-    # Gets all similar tokens and puts together the possible combinations
-    # for each found similar token.
-    # 
-    def similar_possible_for token
-      # Get as many similar tokens as necessary
-      #
-      tokens = similar_tokens_for token
-      # possible combinations
-      #
-      inject_possible_for tokens
-    end
-    def similar_tokens_for token
-      text = token.text
-      categories.inject([]) do |result, category|
-        next_token = token
-        # Note: We could also break off here if not all the available
-        #       similars are needed.
-        #       Wait for a concrete case that needs this before taking
-        #       action.
-        #
-        while next_token = next_token.next_similar_token(category)
-          result << next_token if next_token && next_token.text != text
-        end
-        result
-      end
-    end
-    def inject_possible_for tokens
-      tokens.inject([]) do |result, token|
-        possible = possible_categories token
-        result + possible_for(token, possible)
-      end
-    end
-    
-    # Returns possible Combinations for the token.
-    #
-    # Note: The preselected_categories param is an optimization.
-    #
-    # Note: Returns [] if no categories matched (will produce no result).
-    #       Returns nil if this token needs to be removed from the query.
-    #       (Also none of the categories matched, but the ignore unassigned
-    #       tokens option is true)
-    #
-    def possible_for token, preselected_categories = nil
-      possible = (preselected_categories || possible_categories(token)).map { |category| category.combination_for(token) }
-      possible.compact!
-      # This is an optimization to mark tokens that are ignored.
-      #
-      return if ignore_unassigned_tokens && possible.empty?
-      possible # wrap in combinations
-    end
-    # This returns the possible categories for this token.
-    # If the user has already preselected a category for this token,
-    # like "artist:moby", if not just return all for the given token,
-    # since all are possible.
-    #
-    # Note: Once I thought this was called too often. But it is not (18.01.2011).
-    #
-    def possible_categories token
-      user_defined_categories(token) || categories
-    end
-    # This returns the array of categories if the user has defined
-    # an existing category.
-    #
-    # Note: Returns nil if the user did not define one
-    #       or if he/she has defined a non-existing one. 
-    #
-    def user_defined_categories token
-      category_hash[token.user_defined_category_name]
-    end
-    
-  end
-  
-end