picky 3.6.7 → 3.6.8

data/lib/picky/backends/sqlite/value.rb

@@ -0,0 +1,34 @@
+module Picky
+
+  module Backends
+
+    class SQLite
+
+      class Value < Basic
+
+        def []= key, value
+          db.execute 'insert or replace into key_value (key, value) values (?,?)',
+                     key.to_s,
+                     Yajl::Encoder.encode(value)
+
+          value
+        end
+
+        def [] key
+          res = db.execute "select value from key_value where key = ? limit 1;", key.to_s
+          return nil if res.empty?
+
+          Yajl::Parser.parse res.first.first
+        end
+
+        def delete key
+          db.execute "delete from key_value where key = (?)", key.to_s
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/picky/backends/sqlite.rb

@@ -4,8 +4,11 @@ module Picky
 
     class SQLite < Backend
 
+      attr_reader :self_indexed
+
       def initialize options = {}
         super options
+        @self_indexed = options[:self_indexed]
 
         require 'sqlite3'
       rescue LoadError => e
@@ -17,28 +20,35 @@ module Picky
       #
       def create_inverted bundle
         extract_lambda_or(inverted, bundle) ||
-        DB.new(bundle.index_path(:inverted))
+          Array.new(bundle.index_path(:inverted), self_indexed: self_indexed)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         extract_lambda_or(weights, bundle) ||
-        DB.new(bundle.index_path(:weights))
+          Value.new(bundle.index_path(:weights), self_indexed: self_indexed)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         extract_lambda_or(similarity, bundle) ||
-        DB.new(bundle.index_path(:similarity))
+          Array.new(bundle.index_path(:similarity), self_indexed: self_indexed)
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle
         extract_lambda_or(configuration, bundle) ||
-        DB.new(bundle.index_path(:configuration))
+          Value.new(bundle.index_path(:configuration), self_indexed: self_indexed)
+      end
+      # Returns an object that on #initial, #load returns an object that responds to:
+      #   [id] # => [:sym1, :sym2]
+      #
+      def create_realtime bundle
+        extract_lambda_or(similarity, bundle) ||
+          Array.new(bundle.index_path(:realtime), self_indexed: self_indexed)
       end
 
       # Returns the result ids for the allocation.

data/lib/picky/bundle.rb

@@ -29,6 +29,7 @@ module Picky
                   :weights,
                   :similarity,
                   :configuration,
+                  :realtime,
 
                   :backend_inverted,
                   :backend_weights,
@@ -73,12 +74,11 @@ module Picky
     def reset_backend
       # Extract specific indexes from backend.
       #
-      # TODO Clean up all related.
-      #
       @backend_inverted      = backend.create_inverted      self
       @backend_weights       = backend.create_weights       self
       @backend_similarity    = backend.create_similarity    self
       @backend_configuration = backend.create_configuration self
+      @backend_realtime      = backend.create_realtime      self
 
       # Initial indexes.
       #
@@ -89,8 +89,7 @@ module Picky
       @weights       = @weights_strategy.saved?? @backend_weights.initial : @weights_strategy
       @similarity    = @backend_similarity.initial
       @configuration = @backend_configuration.initial
-
-      @realtime_mapping = {} # id -> ary of syms.  TODO Always instantiate? d
+      @realtime      = @backend_realtime.initial
     end
 
     # "Empties" the index(es) by getting a new empty
@@ -101,6 +100,7 @@ module Picky
       empty_weights
       empty_similarity
       empty_configuration
+      empty_realtime
     end
     def empty_inverted
       @inverted = @backend_inverted.empty
@@ -116,6 +116,9 @@ module Picky
     def empty_configuration
       @configuration = @backend_configuration.empty
     end
+    def empty_realtime
+      @realtime = @backend_realtime.empty
+    end
 
     # Get a list of similar texts.
     #
@@ -124,7 +127,10 @@ module Picky
     def similar text
       code = similarity_strategy.encoded text
       similar_codes = code && @similarity[code]
-      similar_codes.delete text if similar_codes
+      if similar_codes
+        similar_codes = similar_codes.dup # TODO
+        similar_codes.delete text
+      end
       similar_codes || []
     end
 
@@ -158,6 +164,7 @@ module Picky
       @backend_weights.delete        if @backend_weights.respond_to?(:delete) && @weights_strategy.saved?
       @backend_similarity.delete     if @backend_similarity.respond_to? :delete
       @backend_configuration.delete  if @backend_configuration.respond_to? :delete
+      @backend_realtime.delete  if @backend_realtime.respond_to? :delete
     end
 
     def to_s

data/lib/picky/bundle_indexed.rb

@@ -16,12 +16,13 @@ module Picky
   #
   class Bundle
 
-    attr_reader :realtime_mapping
-
     # Get the ids for the given symbol.
     #
     # Returns a (potentially empty) array of ids.
     #
+    # TODO Empty string ok, or does it need to be from the backend?
+    #      Should the backend always return an empty array? (Probably no)
+    #
     def ids sym_or_string
       @inverted[sym_or_string] || []
     end
@@ -52,6 +53,7 @@ module Picky
       load_weights
       load_similarity
       load_configuration
+      load_realtime
     end
 
     # Loads the core index.
@@ -76,6 +78,11 @@ module Picky
     def load_configuration
       self.configuration = @backend_configuration.load
     end
+    # Loads the realtime mapping.
+    #
+    def load_realtime
+      self.realtime = @backend_realtime.load
+    end
 
     # Clears all indexes.
     #
@@ -84,6 +91,7 @@ module Picky
       clear_weights
       clear_similarity
       clear_configuration
+      clear_realtime
     end
 
     # Clears the core index.
@@ -106,6 +114,11 @@ module Picky
     def clear_configuration
       configuration.clear
     end
+    # Clears the realtime mapping.
+    #
+    def clear_realtime
+      realtime.clear
+    end
 
   end
 

data/lib/picky/bundle_indexing.rb

@@ -28,11 +28,6 @@ module Picky
   #
   class Bundle
 
-    # When indexing, clear only clears the inverted index.
-    #
-    delegate :clear,
-             :to => :inverted
-
     # Saves the indexes in a dump file.
     #
     def dump
@@ -40,6 +35,7 @@ module Picky
       dump_similarity
       dump_weights
       dump_configuration
+      dump_realtime
     end
     # Dumps the core index.
     #
@@ -63,6 +59,11 @@ module Picky
     def dump_configuration
       @backend_configuration.dump @configuration
     end
+    # Dumps the realtime.
+    #
+    def dump_realtime
+      @backend_realtime.dump @realtime
+    end
 
   end
 

data/lib/picky/bundle_realtime.rb

@@ -7,36 +7,31 @@ module Picky
     def remove id
       # Is it anywhere?
       #
-      syms = @realtime_mapping[id]
+      syms = @realtime[id]
       return unless syms
 
       syms.each do |sym|
         ids = @inverted[sym]
         ids.delete id
 
-        encoded = self.similarity_strategy.encoded sym
-
         if ids.empty?
-          @inverted.delete   sym
-          @weights.delete    sym
+          @inverted.delete sym
+          @weights.delete  sym
           # Since no element uses this sym anymore, we can delete the similarity for it.
           # TODO Not really. Since multiple syms can point to the same encoded.
-          @similarity.delete encoded
+          @similarity.delete self.similarity_strategy.encoded(sym)
         else
           @weights[sym] = self.weights_strategy.weight_for ids.size
         end
       end
 
-      @realtime_mapping.delete id
+      @realtime.delete id
     end
 
     # Returns a reference to the array where the id has been added.
     #
     def add id, str_or_sym, where = :unshift
-      ary = @inverted[str_or_sym]
-
-      str_or_syms = @realtime_mapping[id]
-      str_or_syms = (@realtime_mapping[id] = []) unless str_or_syms # TODO Nicefy.
+      str_or_syms = @realtime[id] || (@realtime[id] = [])
 
       # Inverted.
       #
@@ -46,7 +41,10 @@ module Picky
         ids.send where, id
       else
         str_or_syms << str_or_sym
-        ids = @inverted[str_or_sym] ||= []
+
+        # TODO Introduce a new method?
+        #
+        ids = @inverted[str_or_sym] || (@inverted[str_or_sym] = []) # Ensures that we get an extended Array
         ids.send where, id
       end
 
@@ -69,13 +67,12 @@ module Picky
     #
     def add_similarity str_or_sym, where = :unshift
       if encoded = self.similarity_strategy.encoded(str_or_sym)
-        similarity = @similarity[encoded] ||= []
-        if similarity.include? str_or_sym
-          similarity.delete str_or_sym  # Not completely correct, as others will also be affected, but meh.
-          similarity.send where, str_or_sym #
-        else
-          similarity.send where, str_or_sym
-        end
+        similarity = @similarity[encoded] || (@similarity[encoded] = []) # Ensures that we get an extended Array
+
+        # Not completely correct, as others will also be affected, but meh.
+        #
+        similarity.delete str_or_sym if similarity.include? str_or_sym
+        similarity.send where, str_or_sym
       end
     end
 
@@ -87,29 +84,23 @@ module Picky
       end
     end
 
-    # Clears the realtime mapping.
-    #
-    def clear_realtime_mapping
-      @realtime_mapping.clear
-    end
-
     # Builds the realtime mapping.
     #
     # Note: Experimental feature.
     #
     # TODO Subset of #add. Rewrite.
     #
-    def build_realtime_mapping
-      clear_realtime_mapping
+    def build_realtime
+      clear_realtime
       @inverted.each_pair do |str_or_sym, ids|
         ids.each do |id|
-          str_or_syms = @realtime_mapping[id]
-          str_or_syms = (@realtime_mapping[id] = []) unless str_or_syms # TODO Nicefy.
-          @realtime_mapping[id] << str_or_sym unless str_or_syms.include? str_or_sym
+          str_or_syms = @realtime[id]
+          str_or_syms = (@realtime[id] = []) unless str_or_syms # TODO Nicefy.
+          @realtime[id] << str_or_sym unless str_or_syms.include? str_or_sym
         end
       end
     end
 
   end
 
-end
+end

data/lib/picky/categories_realtime.rb

@@ -5,7 +5,7 @@ module Picky
     each_delegate :remove,
                   :add,
                   :replace,
-                  :clear_realtime_mapping,
+                  :clear_realtime,
                   :build_realtime_mapping,
                   :to => :categories
 

data/lib/picky/category_indexed.rb

@@ -8,7 +8,7 @@ module Picky
     #
     def load
       timed_exclaim %Q{"#{identifier}": Loading index from cache.}
-      clear_realtime_mapping # TODO What to do?
+      clear_realtime # TODO What to do?
       exact.load
       partial.load
     end

data/lib/picky/category_indexing.rb

@@ -51,7 +51,7 @@ module Picky
       empty
       retrieve
       dump
-      clear_realtime_mapping # TODO To call or not to call, that is the question.
+      clear_realtime # TODO To call or not to call, that is the question.
     end
 
     # Retrieves the prepared index data into the indexes and
@@ -107,12 +107,14 @@ module Picky
       @tokenizer || @index.tokenizer
     end
 
-    # Deletes the caches.
+    # Clears the caches.
+    #
+    # TODO Think about the semantics of clear.
+    #      Is a delete even needed or is it clear+dump?
     #
     def clear
-      timed_exclaim "Deleting #{identifier}."
-      exact.delete
-      partial.delete
+      exact.clear
+      partial.clear
     end
 
   end

data/lib/picky/category_realtime.rb

@@ -27,6 +27,18 @@ module Picky
       add object, where
     end
 
+    # Add at the end.
+    #
+    def << thing
+      add thing, __method__
+    end
+
+    # Add at the beginning.
+    #
+    def unshift thing
+      add thing, __method__
+    end
+
     # For the given id, adds the list of
     # strings to the index for the given id.
     #
@@ -47,16 +59,16 @@ module Picky
 
     # Clears the realtime mapping.
     #
-    def clear_realtime_mapping
-      exact.clear_realtime_mapping
-      partial.clear_realtime_mapping
+    def clear_realtime
+      exact.clear_realtime
+      partial.clear_realtime
     end
 
     # Builds the realtime mapping.
     #
     def build_realtime_mapping
-      exact.build_realtime_mapping
-      partial.build_realtime_mapping
+      exact.build_realtime
+      partial.build_realtime
     end
 
   end

data/lib/picky/generators/strategy.rb

@@ -13,6 +13,10 @@ module Picky
         true
       end
 
+      def to_s
+        self.class
+      end
+
     end
 
   end

data/lib/picky/index_indexing.rb

@@ -87,7 +87,7 @@ module Picky
     #
     def source some_source = nil, &block
       some_source ||= block
-      some_source ? define_source(some_source) : (@source && extract_source || warn_no_source)
+      some_source ? define_source(some_source) : (@source && extract_source)
     end
     # Extract the actual source if it is wrapped in a time
     # capsule, i.e. a block/lambda.
@@ -101,9 +101,6 @@ module Picky
       check_source source
       @source = source
     end
-    def warn_no_source
-      warn "No source given for index #{name}."
-    end
     def check_source source # :nodoc:
       raise ArgumentError.new(<<-SOURCE
 

data/lib/picky/index_realtime.rb

@@ -4,15 +4,25 @@ module Picky
   #
   class Index
 
-    # TODO Rake troubles?
-    #
-    delegate :remove,
-             :add,
-             :replace,
-             :clear_realtime_mapping,
+    delegate :remove,  # aka "delete".
+             :add,     # aka "insert".
+             :replace, # aka "insert or update".
+             :clear_realtime,
              :build_realtime_mapping,
              :to => :categories
 
+    # Add at the end.
+    #
+    def << thing
+      add thing, __method__
+    end
+
+    # Add at the beginning (calls add).
+    #
+    def unshift thing
+      add thing, __method__
+    end
+
   end
 
 end

data/lib/picky/indexers/base.rb

@@ -10,7 +10,8 @@ module Picky
 
       attr_reader :index_or_category
 
-      delegate :source, :to => :index_or_category
+      delegate :source,
+               :to => :index_or_category
 
       def initialize index_or_category
         @index_or_category = index_or_category
@@ -19,6 +20,7 @@ module Picky
       # Starts the indexing process.
       #
       def index categories
+        check_source
         categories.empty
         process categories do |file|
           notify_finished file
@@ -26,6 +28,10 @@ module Picky
         categories.cache
       end
 
+      def check_source # :nodoc:
+        raise "Trying to index without a source for #{@index_or_category.name}." unless source
+      end
+
       def notify_finished file
         timed_exclaim %Q{"#{@index_or_category.identifier}": Tokenized -> #{file.path.gsub("#{PICKY_ROOT}/", '')}.}
       end

data/lib/picky/indexes.rb

@@ -33,6 +33,7 @@ module Picky
     end
 
     instance_delegate :clear,
+                      :clear_indexes,
                       :register,
                       :reindex,
                       :[],

data/lib/picky/loader.rb

@@ -112,12 +112,6 @@ module Picky
       load_relative 'backends/helpers/file'
       load_relative 'backends/backend'
 
-      load_relative 'backends/redis'
-      load_relative 'backends/redis/basic'
-      load_relative 'backends/redis/list'
-      load_relative 'backends/redis/string'
-      load_relative 'backends/redis/float'
-
       load_relative 'backends/memory'
       load_relative 'backends/memory/basic'
       load_relative 'backends/memory/text'
@@ -128,8 +122,18 @@ module Picky
       load_relative 'backends/file/basic'
       load_relative 'backends/file/json'
 
+      load_relative 'backends/redis'
+      load_relative 'backends/redis/directly_manipulable'
+      load_relative 'backends/redis/basic'
+      load_relative 'backends/redis/list'
+      load_relative 'backends/redis/string'
+      load_relative 'backends/redis/float'
+
       load_relative 'backends/sqlite'
-      load_relative 'backends/sqlite/db'
+      load_relative 'backends/sqlite/directly_manipulable'
+      load_relative 'backends/sqlite/basic'
+      load_relative 'backends/sqlite/array'
+      load_relative 'backends/sqlite/value'
 
       # Indexing and Indexed things.
       #

data/lib/picky/query/allocation.rb

@@ -18,7 +18,7 @@ module Picky
       def initialize index, combinations
         @combinations      = combinations
         @result_identifier = index.result_identifier # TODO Make cleverer.
-        @backend           = index.backend           # TODO Make cleverer.
+        @backend           = index.backend           # TODO Make cleverer. Use inverted?
       end
 
       def hash

data/lib/picky/query/indexes.rb

@@ -54,7 +54,7 @@ module Picky
 
       # Returns a number of prepared (sorted, reduced etc.) allocations for the given tokens.
       #
-      def prepared_allocations_for tokens, weights = {}
+      def prepared_allocations_for tokens, weights = {}, amount = nil
         allocations = allocations_for tokens
 
         # Remove double allocations.
@@ -72,7 +72,7 @@ module Picky
 
         # Reduce the amount of allocations.
         #
-        # allocations.reduce_to some_amount
+        allocations.reduce_to amount if amount
 
         # Remove categories from allocations.
         #

data/lib/picky/query/token.rb

@@ -132,7 +132,7 @@ module Picky
       #       (We avoid a StopIteration exception. Which of both is less evil?)
       #
       def generate_similarity_for bundle
-        bundle.similar(@text).dup || []
+        bundle.similar(@text) || []
       end
 
       # Splits text into a qualifier and text.

data/lib/picky/search.rb

@@ -49,6 +49,8 @@ module Picky
       self
     end
 
+    # Defines tokenizer options or the tokenizer itself.
+    #
     # Examples:
     #   search = Search.new(index1, index2, index3) do
     #     searching removes_characters: /[^a-z]/,
@@ -67,6 +69,17 @@ module Picky
       end
     end
 
+    # Sets the max amount of allocations to calculate.
+    #
+    # Examples:
+    #   search = Search.new(index1, index2, index3) do
+    #     max_allocations 10
+    #   end
+    #
+    def max_allocations amount = nil
+      amount ? @max_allocations = amount : @max_allocations
+    end
+
     # Examples:
     #   search = Search.new(books_index, dvd_index, mp3_index) do
     #     boost [:author, :title] => +3,
@@ -161,7 +174,7 @@ module Picky
     # Note: Internal method, use #search to search.
     #
     def execute tokens, ids, offset, original_text = nil
-      Results.from original_text, ids, offset, sorted_allocations(tokens)
+      Results.from original_text, ids, offset, sorted_allocations(tokens, @max_allocations)
     end
 
     # Delegates the tokenizing to the query tokenizer.
@@ -182,19 +195,18 @@ module Picky
 
     # Gets sorted allocations for the tokens.
     #
-    def sorted_allocations tokens # :nodoc:
-      indexes.prepared_allocations_for tokens, weights
+    def sorted_allocations tokens, amount = nil # :nodoc:
+      indexes.prepared_allocations_for tokens, weights, amount
     end
 
     # Display some nice information for the user.
     #
     def to_s
       s = "#{self.class}("
-      unless @indexes.indexes.empty?
-        s << @indexes.indexes.map(&:name).join(', ')
-        s << ", "
-      end
-      s << "weights: #{@weights}"
+      ary = []
+      ary << @indexes.indexes.map(&:name).join(', ') unless @indexes.indexes.empty?
+      ary << "weights: #{@weights}" if @weights
+      s << ary.join(', ')
       s << ")"
       s
     end