picky 3.2.0 → 3.3.0

data/aux/picky/cli.rb

@@ -100,8 +100,10 @@ module Picky
 
     # Maps commands to the other gem's command.
     #
+    # TODO Try to load the other gems and get the commands dynamically.
+    #
     @@mapping = {
-      :generate => [Generate, :'{sinatra_client,unicorn_server,empty_unicorn_server}', :'app_directory_name'],
+      :generate => [Generate, :'{client,server,sinatra_client,unicorn_server,all_in_one}', :'app_directory_name'],
       :help     => [Help],
       :live     => [Live, 'host:port/path (default: localhost:8080/admin)', 'port (default: 4568)'],
       :search   => [Search, :url_or_path, 'amount of ids (default 20)'],

data/lib/picky/backends/backend.rb

@@ -4,6 +4,22 @@ module Picky
 
     class Backend
 
+      attr_reader :inverted,
+                  :weights,
+                  :similarity,
+                  :configuration
+
+      def initialize options = {}
+        @inverted      = options[:inverted]
+        @weights       = options[:weights]
+        @similarity    = options[:similarity]
+        @configuration = options[:configuration]
+      end
+
+      def extract_lambda_or thing, *args
+        thing && (thing.respond_to?(:call) && thing.call(*args) || thing)
+      end
+
       #
       #
       def to_s

data/lib/picky/backends/file/basic.rb

@@ -21,13 +21,16 @@ module Picky
         # An index cache takes a path, without file extension,
         # which will be provided by the subclasses.
         #
-        def initialize cache_path
+        def initialize cache_path, options = {}
           @cache_path = "#{cache_path}.file.#{extension}"
 
           # This is the mapping file with the in-memory hash for the
           # file position/offset mappings.
           #
           @mapping_file = Memory::JSON.new "#{cache_path}.file_mapping.#{extension}"
+
+          @empty   = options[:empty]
+          @initial = options[:initial]
         end
 
         # The default extension for index files is "index".
@@ -36,20 +39,20 @@ module Picky
           :index
         end
 
+        # The empty index that is used for putting the index
+        # together before it is saved into the files.
+        #
+        def empty
+          @empty && @empty.clone || {}
+        end
+
         # The initial content before loading.
         #
         # Note: We could also load the mapping file
         #       as in #load.
         #
         def initial
-          nil
-        end
-
-        # The empty index that is used for putting the index
-        # together before it is saved into the files.
-        #
-        def empty
-          {}
+          @initial && @initial.clone || nil
         end
 
         # Will copy the index file to a location that
@@ -96,6 +99,12 @@ module Picky
           size_of(cache_path) > 0
         end
 
+        #
+        #
+        def to_s
+          "#{self.class}(#{cache_path},#{mapping_file.cache_path})"
+        end
+
       end
 
     end

data/lib/picky/backends/file/json.rb

@@ -52,6 +52,7 @@ module Picky
           offset = 0
           mapping = {}
 
+          create_directory cache_path
           ::File.open(cache_path, 'w:utf-8') do |out_file|
             hash.each do |(key, object)|
               encoded = Yajl::Encoder.encode object

data/lib/picky/backends/file.rb

@@ -14,25 +14,29 @@ module Picky
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
-        JSON.new bundle.index_path(:inverted)
+        extract_lambda_or(inverted, bundle) ||
+        JSON.new(bundle.index_path(:inverted))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
-        JSON.new bundle.index_path(:weights)
+        extract_lambda_or(weights, bundle) ||
+        JSON.new(bundle.index_path(:weights))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
-        JSON.new bundle.index_path(:similarity)
+        extract_lambda_or(similarity, bundle) ||
+        JSON.new(bundle.index_path(:similarity))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
-        JSON.new bundle.index_path(:configuration)
+        extract_lambda_or(configuration, bundle) ||
+        JSON.new(bundle.index_path(:configuration))
       end
 
       # Currently, the loaded ids are intersected using

data/lib/picky/backends/helpers/file.rb

@@ -8,6 +8,12 @@ module Picky
       #
       module File
 
+        # Creates all necessary directories.
+        #
+        def create_directory path
+          FileUtils.mkdir_p ::File.dirname(path)
+        end
+
         # The backup directory of this file.
         # Equal to the file's dirname plus /backup
         #

data/lib/picky/backends/memory/basic.rb

@@ -22,8 +22,10 @@ module Picky
         # An index cache takes a path, without file extension,
         # which will be provided by the subclasses.
         #
-        def initialize cache_path
+        def initialize cache_path, options = {}
           @cache_path = "#{cache_path}.memory.#{extension}"
+          @empty      = options[:empty]
+          @initial    = options[:initial]
         end
 
         # The default extension for index files is "index".
@@ -36,7 +38,13 @@ module Picky
         # together before it is dumped into the files.
         #
         def empty
-          {}
+          @empty && @empty.clone || {}
+        end
+
+        # The initial content before loading from file.
+        #
+        def initial
+          @initial && @initial.clone || {}
         end
 
         # Will copy the index file to a location that

data/lib/picky/backends/memory/json.rb

@@ -14,12 +14,6 @@ module Picky
           :json
         end
 
-        # The initial content before loading.
-        #
-        def initial
-          {}
-        end
-
         # Loads the index hash from json format.
         #
         def load
@@ -29,6 +23,7 @@ module Picky
         # Dumps the index hash in json format.
         #
         def dump hash
+          create_directory cache_path
           hash.dump_json cache_path
         end
 

data/lib/picky/backends/memory/marshal.rb

@@ -14,12 +14,6 @@ module Picky
           :dump
         end
 
-        # The initial content before loading.
-        #
-        def initial
-          {}
-        end
-
         # Loads the index hash from marshal format.
         #
         def load
@@ -29,6 +23,7 @@ module Picky
         # Dumps the index hash in marshal format.
         #
         def dump hash
+          create_directory cache_path
           hash.dump_marshal cache_path
         end
 

data/lib/picky/backends/memory/text.rb

@@ -57,6 +57,7 @@ module Picky
         #
         #
         def open &block
+          create_directory cache_path
           ::File.open cache_path, 'w:utf-8', &block
         end
 

data/lib/picky/backends/memory.rb

@@ -8,25 +8,29 @@ module Picky
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
-        JSON.new bundle.index_path(:inverted)
+        extract_lambda_or(inverted, bundle) ||
+        JSON.new(bundle.index_path(:inverted))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
-        JSON.new bundle.index_path(:weights)
+        extract_lambda_or(weights, bundle) ||
+        JSON.new(bundle.index_path(:weights))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
-        Marshal.new bundle.index_path(:similarity)
+        extract_lambda_or(similarity, bundle) ||
+        Marshal.new(bundle.index_path(:similarity))
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
-        JSON.new bundle.index_path(:configuration)
+        extract_lambda_or(configuration, bundle) ||
+        JSON.new(bundle.index_path(:configuration))
       end
 
       # Returns the result ids for the allocation.

data/lib/picky/backends/redis/basic.rb

@@ -18,9 +18,19 @@ module Picky
         # An index cache takes a path, without file extension,
         # which will be provided by the subclasses.
         #
-        def initialize client, namespace
+        def initialize client, namespace, options = {}
           @client    = client
           @namespace = namespace
+
+          @empty     = options[:empty]
+          @initial   = options[:initial]
+        end
+
+        # The empty index that is used for putting the index
+        # together.
+        #
+        def empty
+          @empty && @empty.clone || {}
         end
 
         # The initial content before loading.
@@ -29,7 +39,7 @@ module Picky
         #       this just returns the same thing as #load.
         #
         def initial
-          self
+          @initial && @initial.clone || self
         end
 
         # Returns itself.
@@ -38,13 +48,6 @@ module Picky
           self
         end
 
-        # The empty index that is used for putting the index
-        # together.
-        #
-        def empty
-          {}
-        end
-
         # We do not use Redis to retrieve data.
         #
         def retrieve

data/lib/picky/backends/redis.rb

@@ -9,6 +9,8 @@ module Picky
       attr_reader :client
 
       def initialize options = {}
+        super options
+
         require 'redis'
         @client = options[:client] || ::Redis.new(:db => (options[:db] || 15))
       rescue LoadError => e
@@ -19,25 +21,29 @@ module Picky
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
-        List.new client, "#{bundle.identifier}:inverted"
+        extract_lambda_or(inverted, client, bundle) ||
+        List.new(client, "#{bundle.identifier}:inverted")
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
-        Float.new client, "#{bundle.identifier}:weights"
+        extract_lambda_or(weights, client, bundle) ||
+        Float.new(client, "#{bundle.identifier}:weights")
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
-        List.new client, "#{bundle.identifier}:similarity"
+        extract_lambda_or(similarity, client, bundle) ||
+        List.new(client, "#{bundle.identifier}:similarity")
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
-        String.new client, "#{bundle.identifier}:configuration"
+        extract_lambda_or(configuration, client, bundle) ||
+        String.new(client, "#{bundle.identifier}:configuration")
       end
 
       # Returns the result ids for the allocation.

data/lib/picky/bundle.rb

@@ -47,6 +47,11 @@ module Picky
       @name     = name
       @category = category
 
+      # TODO Tidy up a bit.
+      #
+      @key_format = options[:key_format]
+      @prepared   = Backends::Memory::Text.new category.prepared_index_path
+
       @weights_strategy    = weights_strategy
       @partial_strategy    = partial_strategy
       @similarity_strategy = similarity_strategy
@@ -59,6 +64,15 @@ module Picky
       @backend_weights       = backend.create_weights       self
       @backend_similarity    = backend.create_similarity    self
       @backend_configuration = backend.create_configuration self
+
+      # Initial indexes.
+      #
+      @inverted      = @backend_inverted.initial
+      @weights       = @backend_weights.initial
+      @similarity    = @backend_similarity.initial
+      @configuration = @backend_configuration.initial
+
+      @realtime_mapping = {} # id -> ary of syms.  TODO Always instantiate?
     end
     def identifier
       "#{category.identifier}:#{name}"

data/lib/picky/bundle_indexed.rb

@@ -0,0 +1,110 @@
+module Picky
+
+  # An indexed bundle is a number of memory/redis
+  # indexes that compose the indexes for a single category:
+  #  * core (inverted) index
+  #  * weights index
+  #  * similarity index
+  #  * index configuration
+  #
+  # Indexed refers to them being indexed.
+  # This class notably offers the methods:
+  #  * load
+  #  * clear
+  #
+  # To (re)load or clear the current indexes.
+  #
+  class Bundle
+
+    attr_reader :realtime_mapping
+
+    # Get the ids for the given symbol.
+    #
+    # Returns a (potentially empty) array of ids.
+    #
+    def ids sym
+      @inverted[sym] || []
+    end
+
+    # Get a weight for the given symbol.
+    #
+    # Returns a number, or nil.
+    #
+    def weight sym
+      @weights[sym]
+    end
+
+    # Get settings for this bundle.
+    #
+    # Returns an object.
+    #
+    def [] sym
+      @configuration[sym]
+    end
+
+    # Loads all indexes.
+    #
+    # Loading loads index objects from the backend.
+    # They should each respond to [] and return something appropriate.
+    #
+    def load
+      load_inverted
+      load_weights
+      load_similarity
+      load_configuration
+    end
+
+    # Loads the core index.
+    #
+    def load_inverted
+      self.inverted = @backend_inverted.load
+    end
+    # Loads the weights index.
+    #
+    def load_weights
+      self.weights = @backend_weights.load
+    end
+    # Loads the similarity index.
+    #
+    def load_similarity
+      self.similarity = @backend_similarity.load
+    end
+    # Loads the configuration.
+    #
+    def load_configuration
+      self.configuration = @backend_configuration.load
+    end
+
+    # Clears all indexes.
+    #
+    def clear
+      clear_inverted
+      clear_weights
+      clear_similarity
+      clear_configuration
+    end
+
+    # Clears the core index.
+    #
+    def clear_inverted
+      inverted.clear
+    end
+    # Clears the weights index.
+    #
+    def clear_weights
+      weights.clear
+    end
+    # Clears the similarity index.
+    #
+    def clear_similarity
+      similarity.clear
+    end
+    # Clears the configuration.
+    #
+    def clear_configuration
+      configuration.clear
+    end
+
+  end
+
+end

data/lib/picky/bundle_indexing.rb

@@ -0,0 +1,177 @@
+module Picky
+
+  # A Bundle is a number of indexes
+  # per [index, category] combination.
+  #
+  # At most, there are three indexes:
+  # * *core* index (always used)
+  # * *weights* index (always used)
+  # * *similarity* index (used with similarity)
+  #
+  # In Picky, indexing is separated from the index
+  # handling itself through a parallel structure.
+  #
+  # Both use methods provided by this base class, but
+  # have very different goals:
+  #
+  # * *Indexing*::*Bundle* is just concerned with creating index files
+  #   and providing helper functions to e.g. check the indexes.
+  #
+  # * *Index*::*Bundle* is concerned with loading these index files into
+  #   memory and looking up search data as fast as possible.
+  #
+  # This is the indexing bundle.
+  #
+  # It does all menial tasks that have nothing to do
+  # with the actual index running etc.
+  # (Find these in Indexed::Bundle)
+  #
+  class Bundle
+
+    attr_reader :backend,
+                :prepared
+
+    # When indexing, clear only clears the inverted index.
+    #
+    delegate :clear, :to => :inverted
+
+    # Sets up a piece of the index for the given token.
+    #
+    def initialize_inverted_index_for token
+      self.inverted[token] ||= []
+    end
+
+    # Generation
+    #
+
+    # This method
+    # * Loads the base index from the "prepared..." file.
+    # * Generates derived indexes.
+    # * Dumps all the indexes into files.
+    #
+    def generate_caches_from_source
+      load_from_prepared_index_file
+      generate_caches_from_memory
+    end
+    # Generates derived indexes from the index and dumps.
+    #
+    # Note: assumes that there is something in the index
+    #
+    def generate_caches_from_memory
+      cache_from_memory_generation_message
+      generate_derived
+    end
+    def cache_from_memory_generation_message
+      timed_exclaim %Q{"#{identifier}": Caching from intermediate in-memory index.}
+    end
+
+    # Generates the weights and similarity from the main index.
+    #
+    def generate_derived
+      generate_weights
+      generate_similarity
+    end
+
+    # "Empties" the index(es) by getting a new empty
+    # internal backend instance.
+    #
+    def empty
+      empty_inverted
+      empty_configuration
+    end
+    def empty_inverted
+      @inverted = @backend_inverted.empty
+    end
+    def empty_configuration
+      @configuration = @backend_configuration.empty
+    end
+
+    # Load the data from the db.
+    #
+    def load_from_prepared_index_file
+      load_from_prepared_index_generation_message
+      retrieve
+    end
+    def load_from_prepared_index_generation_message
+      timed_exclaim %Q{"#{identifier}": Loading prepared data into memory.}
+    end
+    # Retrieves the prepared index data into the index.
+    #
+    # This is in preparation for generating
+    # derived indexes (like weights, similarity)
+    # and later dumping the optimized index.
+    #
+    # TODO Move this out to the category?
+    #
+    def retrieve
+      format = key_format || :to_i
+      empty_inverted
+      prepared.retrieve do |id, token|
+        initialize_inverted_index_for token
+        self.inverted[token] << id.send(format)
+      end
+    end
+
+    # Generates a new index (writes its index) using the
+    # partial caching strategy of this bundle.
+    #
+    def generate_partial
+      generator = Generators::PartialGenerator.new self.inverted
+      self.inverted = generator.generate self.partial_strategy
+    end
+    # Generate a partial index from the given exact inverted index.
+    #
+    def generate_partial_from exact_inverted_index
+      timed_exclaim %Q{"#{identifier}": Generating partial index for index.}
+      self.inverted = exact_inverted_index
+      self.generate_partial
+      self
+    end
+    # Generates a new weights index (writes its index) using the
+    # given weight caching strategy.
+    #
+    def generate_weights
+      generator = Generators::WeightsGenerator.new self.inverted
+      self.weights = generator.generate self.weights_strategy
+    end
+    # Generates a new similarity index (writes its index) using the
+    # given similarity caching strategy.
+    #
+    def generate_similarity
+      generator = Generators::SimilarityGenerator.new self.inverted
+      self.similarity = generator.generate self.similarity_strategy
+    end
+
+    # Saves the indexes in a dump file.
+    #
+    def dump
+      timed_exclaim %Q{"#{identifier}": Dumping data.}
+      dump_inverted
+      dump_similarity
+      dump_weights
+      dump_configuration
+    end
+    # Dumps the core index.
+    #
+    def dump_inverted
+      @backend_inverted.dump self.inverted
+    end
+    # Dumps the weights index.
+    #
+    def dump_weights
+      @backend_weights.dump self.weights
+    end
+    # Dumps the similarity index.
+    #
+    def dump_similarity
+      @backend_similarity.dump self.similarity
+    end
+    # Dumps the similarity index.
+    #
+    def dump_configuration
+      @backend_configuration.dump self.configuration
+    end
+
+  end
+
+end