picky 4.26.2 → 4.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3a6133019bb7fb372e2ef9a8261ee69df1e562b
-  data.tar.gz: 8dfb39edf77042b40276bc0e26d34286c3e71b79
+  metadata.gz: 20377f0b4ac92cd75659306842de34204b7bf85d
+  data.tar.gz: 7c3e252193f4be20dd6fb5013b11a9d427662051
 SHA512:
-  metadata.gz: d21d378abbbd2ea37ca0eb6ce1e948c5904db3daa6f6df41797a6acaff996e12fcf43cf5c5cc536694d0883c5918ca6a1049fe22eed369ba563f495688999cef
-  data.tar.gz: 50ca4ff1812761a21af2c5aaa781f75bddfd4fbe20a8de72334f660a826eba3c551933c56924e2415af7eebd240ce33aea588207700cc38c1a2ea27db04e5573
+  metadata.gz: 02e007ee3b0e7be2a4ce3c337bde4d496c4ccce05450196d8e7135d69d08dec5020d4fa8858b58ae54e170b64fd62677bd08df6c7eb348ada0c78252b7effbd2
+  data.tar.gz: 41285d5f985a0ed59407ba0b8f52526f122a86af8be4f90341b39d79884a2b922d9883a16ae39f0f6ecb997fb5edd1e5208c960d8811dedd09b1c22821c3bb8f

data/lib/picky/backends/backend.rb

@@ -12,37 +12,23 @@ module Picky
       
       # Returns an object that on #initial, #load returns
       # an object that responds to:
-      #   [:token] # => [id, id, id, id, id] (an array of ids)
+      #   object[:token] # => [id, id, id, id, id] (an array of ids)
       #
-      def create_inverted bundle
+      def create_inverted bundle, _ = nil
         json bundle.index_path(:inverted)
       end
       # Returns an object that on #initial, #load returns
       # an object that responds to:
-      #   [:token] # => 1.23 (a weight)
+      #   object[:key] # => value (a value for this config key)
       #
-      # def create_weights bundle
-      #   json bundle.index_path(:weights)
-      # end
-      # # Returns an object that on #initial, #load returns
-      # # an object that responds to:
-      # #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
-      # #
-      # def create_similarity bundle
-      #   Marshal.new bundle.index_path(:similarity)
-      # end
-      # Returns an object that on #initial, #load returns
-      # an object that responds to:
-      #   [:key] # => value (a value for this config key)
-      #
-      def create_configuration bundle
+      def create_configuration bundle, _ = nil
         json bundle.index_path(:configuration)
       end
       # Returns an object that on #initial, #load returns
       # an object that responds to:
-      #   [id] # => [:sym1, :sym2]
+      #   object[id] # => [:sym1, :sym2]
       #
-      def create_realtime bundle
+      def create_realtime bundle, _ = nil
         json bundle.index_path(:realtime)
       end
 

data/lib/picky/backends/file.rb

@@ -10,11 +10,11 @@ module Picky
     #
     class File < Backend
       
-      def create_weights bundle
+      def create_weights bundle, _ = nil
         Memory::JSON.new bundle.index_path(:weights)
       end
       
-      def create_similarity bundle
+      def create_similarity bundle, _ = nil
         json bundle.index_path(:similarity)
       end
 

data/lib/picky/backends/google_hash.rb

@@ -0,0 +1,46 @@
+if require 'google_hash'
+  
+  class ::GoogleHashSparseRubyToRuby
+
+    # This is mainly used in tests, which is
+    # why we allow for a silly implementation.
+    #
+    def == hash
+      return false unless hash.respond_to?(:to_h)
+    
+      self.each do |key, value|
+        return false if hash[key] != value
+      end
+      hash.each do |key, value|
+        return false if self[key] != value
+      end
+    
+      true
+    end
+  
+    # I am a hashy thing.
+    #
+    def to_hash
+      true
+    end
+  
+    #
+    #
+    def inject init, &block
+      result = init
+      each do |key, value|
+        result = block.call result, [key, value]
+      end
+      result
+    end
+  
+    def size
+      result = 0
+      # each only accepts a block
+      each { result += 1 }
+      result
+    end
+
+  end
+  
+end

data/lib/picky/backends/memory/basic.rb

@@ -18,12 +18,17 @@ module Picky
         # This file's location.
         #
         attr_reader :cache_path
+        
+        # What hash type to use. Default: ::Hash
+        #
+        attr_reader :hash_type
 
         # An index cache takes a path, without file extension,
         # which will be provided by the subclasses.
         #
-        def initialize cache_path, options = {}
+        def initialize cache_path, hash_type = Hash, options = {}
           @cache_path = "#{cache_path}.memory.#{extension}"
+          @hash_type  = hash_type
           @empty      = options[:empty]
           @initial    = options[:initial]
         end
@@ -38,7 +43,7 @@ module Picky
         # together before it is dumped into the files.
         #
         def empty
-          @empty && @empty.clone || empty_hash
+          @empty && @empty.clone || hash_type.new
         end
         
         def empty_hash
@@ -50,10 +55,10 @@ module Picky
           end
         end
 
-        # The initial content before loading from file.
+        # The initial content before loading from file/indexing.
         #
         def initial
-          @initial && @initial.clone || {}
+          @initial && @initial.clone || hash_type.new
         end
 
         # Deletes the file.

data/lib/picky/backends/memory.rb

@@ -4,13 +4,58 @@ module Picky
 
     class Memory < Backend
       
-      def create_weights bundle
-        JSON.new bundle.index_path(:weights)
+      # TODO Make lazy.
+      require_relative 'google_hash'
+      
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   object[:token] # => [id, id, id, id, id] (an array of ids)
+      #
+      def create_inverted bundle, hints = nil
+        json bundle.index_path(:inverted), hash_for(hints)
+      end
+      
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   object[:token] # => 1.23 (a weight)
+      #
+      def create_weights bundle, hints = nil
+        JSON.new bundle.index_path(:weights), hash_for(hints) # GoogleHashSparseRubyToInt (I wish they had floats)
+      end
+      
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   object[:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
+      #
+      def create_similarity bundle, hints = nil
+        Marshal.new bundle.index_path(:similarity), hash_for(hints)
       end
       
-      def create_similarity bundle
-        Marshal.new bundle.index_path(:similarity)
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   object[:key] # => value (a value for this config key)
+      #
+      def create_configuration bundle, hints = nil
+        json bundle.index_path(:configuration), hash_for(hints)
       end
+      
+      # Returns an object that on #initial, #load returns
+      # an object that responds to:
+      #   object[id] # => [:sym1, :sym2]
+      #
+      def create_realtime bundle, hints = nil
+        json bundle.index_path(:realtime), hash_for(hints) # GoogleHashSparseLongToRuby # FIXME This is only true on number keys (add Picky hints).
+      end
+      
+      private
+      
+        def hash_for hints
+          if hints && hints.does?(:no_dump)
+            ::GoogleHashSparseRubyToRuby
+          else
+            ::Hash
+          end
+        end
 
     end
 

data/lib/picky/backends/redis.rb

@@ -36,31 +36,31 @@ module Picky
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
-      def create_inverted bundle
+      def create_inverted bundle, _ = nil
         List.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:inverted", realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
-      def create_weights bundle
+      def create_weights bundle, _ = nil
         Float.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:weights", realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
-      def create_similarity bundle
+      def create_similarity bundle, _ = nil
         List.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:similarity", realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
-      def create_configuration bundle
+      def create_configuration bundle, _ = nil
         String.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:configuration", realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [id] # => [:sym1, :sym2]
       #
-      def create_realtime bundle
+      def create_realtime bundle, _ = nil
         List.new client, "#{PICKY_ENVIRONMENT}:#{bundle.identifier}:realtime", realtime: realtime
       end
 

data/lib/picky/backends/sqlite.rb

@@ -17,31 +17,31 @@ module Picky
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
-      def create_inverted bundle
+      def create_inverted bundle, _ = nil
         StringKeyArray.new bundle.index_path(:inverted), realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
-      def create_weights bundle
+      def create_weights bundle, _ = nil
         Value.new bundle.index_path(:weights), realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
-      def create_similarity bundle
+      def create_similarity bundle, _ = nil
         StringKeyArray.new bundle.index_path(:similarity), realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
-      def create_configuration bundle
+      def create_configuration bundle, _ = nil
         Value.new bundle.index_path(:configuration), realtime: realtime
       end
       # Returns an object that on #initial, #load returns an object that responds to:
       #   [id] # => [:sym1, :sym2]
       #
-      def create_realtime bundle
+      def create_realtime bundle, _ = nil
         IntegerKeyArray.new bundle.index_path(:realtime), realtime: realtime
       end
 

data/lib/picky/bundle.rb

@@ -53,7 +53,8 @@ module Picky
       @weight_strategy     = weight_strategy
       @partial_strategy    = partial_strategy
       @similarity_strategy = similarity_strategy
-
+      
+      @hints      = options.delete :hints
       @backend    = options.delete :backend
 
       reset_backend
@@ -81,11 +82,11 @@ module Picky
     # TODO Move @backend_ into the backend?
     #
     def create_backends
-      @backend_inverted      = backend.create_inverted self
-      @backend_weights       = backend.create_weights self
-      @backend_similarity    = backend.create_similarity self
-      @backend_configuration = backend.create_configuration self
-      @backend_realtime      = backend.create_realtime self
+      @backend_inverted      = backend.create_inverted self, @hints
+      @backend_weights       = backend.create_weights self, @hints
+      @backend_similarity    = backend.create_similarity self, @hints
+      @backend_configuration = backend.create_configuration self, @hints
+      @backend_realtime      = backend.create_realtime self, @hints
     end
 
     # Initial indexes.

data/lib/picky/category.rb

@@ -36,6 +36,10 @@ module Picky
     def initialize name, index, options = {}
       @name  = name
       @index = index
+      
+      # TODO Move.
+      #
+      options[:hints] = index.hints
 
       configure_from options
       configure_indexes_from options
@@ -77,7 +81,19 @@ module Picky
     #
     # TODO Rewrite it such that this does not need to be maintained separately (and gets available options automatically).
     #
-    @@known_keys = [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :tokenize, :tokenizer, :weight]
+    @@known_keys = [
+      :hints,
+      :indexing,
+      :partial,
+      :qualifier,
+      :qualifiers,
+      :ranging,
+      :similarity,
+      :source,
+      :tokenize,
+      :tokenizer,
+      :weight,
+    ]
     def warn_if_unknown options
       if options && (options.keys - @@known_keys).size > 0
         warn <<-WARNING

data/lib/picky/index/hints.rb

@@ -0,0 +1,37 @@
+module Picky
+  class Index
+    
+    class Hints
+      
+      def initialize hints
+        @hints = check hints
+      end
+      
+      # Check if all hints are allowed
+      #
+      @@allowed_hints = [:no_dump]
+      def check hints
+        hints.each do |hint|
+          unless @@allowed_hints.include?(hint)
+            raise <<-ERROR
+              Picky cannot optimize for #{hint}.
+              Allowed hints are:
+                #{@@allowed_hints.join("\n")}
+            ERROR
+          end
+        end
+        hints
+      end
+      
+      # Tells us if the user intends to e.g. not dump the indexes.
+      #
+      # E.g. hints.does?(:no_dump) # => true/false
+      #
+      def does? hint
+        @hints.include? hint
+      end
+      
+    end
+    
+  end
+end

data/lib/picky/index.rb

@@ -87,7 +87,8 @@ module Picky
   class Index
 
     attr_reader :name,
-                :categories
+                :categories,
+                :hints
 
     forward :[],
             :dump,
@@ -126,6 +127,13 @@ module Picky
 
       instance_eval(&Proc.new) if block_given?
     end
+    
+    # Provide hints for Picky so it can optimise.
+    #
+    def optimize *hints
+      require_relative 'index/hints'
+      @hints = Hints.new hints
+    end
 
     # TODO Doc.
     #

data/spec/functional/backends/memory_bundle_realtime_spec.rb

@@ -17,17 +17,17 @@ describe Picky::Bundle do
                                         backend: Picky::Backends::Memory.new
   end
 
-  it 'is by default a Hash' do
-    @bundle.realtime.should be_kind_of(Hash)
+  it 'is by default Hash-like' do
+    @bundle.realtime.should respond_to(:to_hash)
   end
-  it 'is by default a Hash' do
-    @bundle.inverted.should be_kind_of(Hash)
+  it 'is by default Hash-like' do
+    @bundle.inverted.should respond_to(:to_hash)
   end
-  it 'is by default a Hash' do
-    @bundle.weights.should be_kind_of(Hash)
+  it 'is by default Hash-like' do
+    @bundle.weights.should respond_to(:to_hash)
   end
-  it 'is by default a Hash' do
-    @bundle.similarity.should be_kind_of(Hash)
+  it 'is by default Hash-like' do
+    @bundle.similarity.should respond_to(:to_hash)
   end
 
   context 'strings' do

data/spec/functional/backends/special_spec.rb

@@ -5,19 +5,19 @@ require 'spec_helper'
 # To test the interface definition.
 #
 class BackendInterfaceTester < Picky::Backends::Backend
-  def create_inverted _
+  def create_inverted _, _ = nil
     InternalBackendInterfaceTester.new
   end
-  def create_weights _
+  def create_weights _, _ = nil
     InternalBackendInterfaceTester.new
   end
-  def create_similarity _
+  def create_similarity _, _ = nil
     InternalBackendInterfaceTester.new
   end
-  def create_configuration _
+  def create_configuration _, _ = nil
     InternalBackendInterfaceTester.new
   end
-  def create_realtime _
+  def create_realtime _, _ = nil
     InternalBackendInterfaceTester.new
   end
 end

data/spec/functional/facets_spec.rb

@@ -174,14 +174,14 @@ describe 'facets' do
       end
       it 'has two filtered facets' do
         finder.facets(:surname, filter: 'age:40 name:peter', counts: false).should == [
-          'kunz',
-          'hanke'
+          'hanke',
+          'kunz'
         ]
       end
       it 'has 2 facets >= count 1' do
         finder.facets(:surname, filter: 'age:40 name:peter', at_least: 1, counts: false).should == [
-          'kunz',
-          'hanke'
+          'hanke',
+          'kunz'
         ]
       end
       it 'has 0 facets >= counts 2' do

data/spec/functional/no_dump_hint_spec.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+require 'ostruct'
+
+describe "Hint: no_dump" do
+
+  Book = Struct.new(:id, :title, :author)
+
+  let(:index) do
+    Picky::Index.new :no_dump do
+      optimize :no_dump
+      
+      category :title
+      category :author
+    end
+  end
+  let(:try) { Picky::Search.new index }
+
+  it 'can index and search' do
+    index.replace Book.new(2, "Title", "Author")
+
+    try.search("title:title").ids.should == [2]
+  end
+  
+  context 'dumping and loading' do
+    it "raises" do
+      index.replace Book.new(2, "Title New", "Author New")
+
+      index.dump
+      index.load
+      index.build_realtime_mapping
+    end
+  end
+
+end

data/spec/lib/backends/memory/basic_spec.rb

@@ -27,6 +27,7 @@ describe Picky::Backends::Memory::Basic do
   context 'with options' do
     let(:basic) do
       described_class.new 'spec/temp/some/cache/path/to/file',
+                          Hash,
                           empty: [],
                           initial: []
     end

data/spec/lib/category_spec.rb

@@ -52,7 +52,7 @@ describe Picky::Category do
       category.should_receive(:warn).once.with <<-WARNING
 
 Warning: Category options {:weights=>:some_weight} for category some_category contain an unknown option.
-         Working options are: [:indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :tokenize, :tokenizer, :weight].
+         Working options are: [:hints, :indexing, :partial, :qualifier, :qualifiers, :ranging, :similarity, :source, :tokenize, :tokenizer, :weight].
 WARNING
       
       category.warn_if_unknown :weights => :some_weight

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: picky
 version: !ruby/object:Gem::Version
-  version: 4.26.2
+  version: 4.27.0
 platform: ruby
 authors:
 - Florian Hanke
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2009.0'
+- !ruby/object:Gem::Dependency
+  name: google_hash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 description: Fast Ruby semantic text search engine with comfortable single field interface.
 email: florian.hanke+picky@gmail.com
 executables:
@@ -73,6 +87,7 @@ files:
 - lib/picky/backends/file.rb
 - lib/picky/backends/file/basic.rb
 - lib/picky/backends/file/json.rb
+- lib/picky/backends/google_hash.rb
 - lib/picky/backends/helpers/file.rb
 - lib/picky/backends/memory.rb
 - lib/picky/backends/memory/basic.rb
@@ -147,6 +162,7 @@ files:
 - lib/picky/helpers/indexing.rb
 - lib/picky/helpers/measuring.rb
 - lib/picky/index.rb
+- lib/picky/index/hints.rb
 - lib/picky/index_convenience.rb
 - lib/picky/index_facets.rb
 - lib/picky/index_indexed.rb
@@ -234,6 +250,7 @@ files:
 - spec/functional/ignore_spec.rb
 - spec/functional/max_allocations_spec.rb
 - spec/functional/multi_index_qualifier_spec.rb
+- spec/functional/no_dump_hint_spec.rb
 - spec/functional/no_tokenize_spec.rb
 - spec/functional/non_specific_ids_larger_than_20_spec.rb
 - spec/functional/object_use_spec.rb
@@ -417,6 +434,7 @@ test_files:
 - spec/functional/ignore_spec.rb
 - spec/functional/max_allocations_spec.rb
 - spec/functional/multi_index_qualifier_spec.rb
+- spec/functional/no_dump_hint_spec.rb
 - spec/functional/no_tokenize_spec.rb
 - spec/functional/non_specific_ids_larger_than_20_spec.rb
 - spec/functional/object_use_spec.rb