picky 4.27.1 → 4.28.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a32ef9ec55047df769e8a264eee585bab738f5d
-  data.tar.gz: 6574e4c5d634313b160d0ff291a4c68e8ee9f553
+  metadata.gz: 1070a597a2893ecafe13461959d34eb72b774fb1
+  data.tar.gz: 187eba56c158b4a3301360c5321423a37e3410af
 SHA512:
-  metadata.gz: ecbb19fb08c6e69ad40bc9a9d6b7416db8ad5b5fc1ac6d340070714cc1e02d70c01ed68fd0de8bd79628af5cb847126983b92c1f847b9bedbd5ffaa2134d85aa
-  data.tar.gz: 9fe9fbcd9109b245777bd48246d466078646a602260e278f73e9c0ba0233e2b69e852903af046412acdebc169072544435d8b2b0eef4d5a9ffdb8f0b400ea693
+  metadata.gz: 399f943b00ce22fffd54c7cb961155fa6386bd898a3d5566f9778c5e883cdb380da9c873eda6644bf1a5d7629e7fe3916e6db64f7049df75f196880e381d0d95
+  data.tar.gz: 7eaa2a8156eb638476fa08edf92617e67ff687096dc631470402f63b7ce91cc0813ee936badd31d4caf7f41bc16e6431e09eeded924ef3b112189224d37df464

data/lib/picky/backends/google_hash.rb

@@ -1,4 +1,5 @@
-if require 'google_hash'
+begin
+  require 'google_hash'
   
   class ::GoogleHashSparseRubyToRuby
     
@@ -47,4 +48,6 @@ if require 'google_hash'
 
   end
   
+rescue LoadError
+  # Welp. Don't do anything.
 end

data/lib/picky/backends/memory/basic.rb

@@ -15,9 +15,9 @@ module Picky
 
         include Helpers::File
 
-        # This file's location.
+        # This file's cache file without extensions.
         #
-        attr_reader :cache_path
+        attr_reader :cache_file_path
         
         # What hash type to use. Default: ::Hash
         #
@@ -26,11 +26,11 @@ module Picky
         # An index cache takes a path, without file extension,
         # which will be provided by the subclasses.
         #
-        def initialize cache_path, hash_type = Hash, options = {}
-          @cache_path = "#{cache_path}.memory.#{extension}"
-          @hash_type  = hash_type
-          @empty      = options[:empty]
-          @initial    = options[:initial]
+        def initialize cache_file_path, hash_type = Hash, options = {}
+          @cache_file_path = cache_file_path
+          @hash_type       = hash_type
+          @empty           = options[:empty]
+          @initial         = options[:initial]
         end
 
         # The default extension for index files is "index".
@@ -38,6 +38,12 @@ module Picky
         def extension
           :index
         end
+        def type
+          :memory
+        end
+        def cache_path
+          [cache_file_path, type, extension].join(?.)
+        end
 
         # The empty index that is used for putting the index
         # together before it is dumped into the files.

data/lib/picky/extensions/string.rb

@@ -27,17 +27,23 @@ class String
   #
   def each_subtoken from_length = 1, range = nil
     sub = self
-
-    sub = sub[range] if range
-
+    
+    if range
+      unless (range.first.zero? && range.last == -1)
+        sub = sub[range].freeze
+      end
+    end
+    
     yield sub
-
+    
     size = sub.size
     from_length = size + from_length + 1 if from_length < 0
     from_length = size if size < from_length
     from_length = 1 if from_length < 1
 
-    size.downto(from_length + 1) { yield sub = sub.chop }
+    size.downto(from_length + 1) { yield sub = sub.chop.freeze }
+    
+    sub = nil
   end
 
   # 'keys'.each_intoken         # => yields each of ['keys', 'key', 'eys', 'ke', 'ey', 'ys', 'k', 'e', 'y', 's']

data/lib/picky/extensions/symbol.rb

@@ -30,7 +30,11 @@ class Symbol
   def each_subtoken from_length = 1, range = nil
     sub = self.id2name
 
-    sub = sub[range] if range
+    if range
+      unless (range.first.zero? && range.last == -1)
+        sub = sub[range]
+      end
+    end
 
     yield sub.intern
 
@@ -39,7 +43,9 @@ class Symbol
     from_length = size if size < from_length
     from_length = 1 if from_length < 1
 
-    size.downto(from_length + 1) { yield sub.chop!.intern }
+    size.downto(from_length + 1) do
+      yield sub.chop!.intern
+    end
   end
 
   # :keys.each_intoken         # => yields each of [:keys, :key, :eys, :ke, :ey, :ys, :k, :e, :y, :s]

data/lib/picky/index.rb

@@ -134,6 +134,14 @@ module Picky
       require_relative 'index/hints'
       @hints = Hints.new hints
     end
+    
+    # Explicitly trigger memory optimization.
+    #
+    def optimize_memory array_references = Hash.new
+      dedup = Picky::Optimizers::Memory::ArrayDeduplicator.new
+      dedup.deduplicate categories.map(&:exact).map(&:inverted), array_references
+      dedup.deduplicate categories.map(&:partial).map(&:inverted), array_references
+    end
 
     # TODO Doc.
     #

data/lib/picky/indexes.rb

@@ -43,6 +43,18 @@ module Picky
       @indexes       = []
       @index_mapping = Hash.new
     end
+    
+    # Tries to optimize the memory usage of the indexes.
+    #
+    def optimize_memory array_references = Hash.new
+      dedup = Picky::Optimizers::Memory::ArrayDeduplicator.new
+      @indexes.each do |index|
+        index.optimize_memory array_references
+      end
+    end
+    def self.optimize_memory array_references = Hash.new
+      self.instance.optimize_memory array_references
+    end
 
     # Registers an index with the indexes.
     #

data/lib/picky/loader.rb

@@ -69,7 +69,9 @@ module Picky
         load_relative 'helpers/measuring',
                       'helpers/indexing',
                       'helpers/identification',
-                      'splitter'
+                      'splitter',
+                      'optimizers',
+                      'optimizers/memory/array_deduplicator'
       end
       def load_index_generation_strategies
         load_relative 'indexers/base',

data/lib/picky/optimizers/memory/array_deduplicator.rb

@@ -0,0 +1,26 @@
+module Picky::Optimizers::Memory
+  
+  # Straightforward implementation of an array deduplicator.
+  # Tries to find duplicate instances of Array values in a hash
+  # and points references that point to a duplicate to one of the
+  # Array instances.
+  #
+  # TODO Could we have C-Ruby point to parts of another Array?
+  #
+  class ArrayDeduplicator
+    
+    def deduplicate hashes, array_references = Hash.new
+      hashes.inject(array_references) do |array_references, hash|
+        deduplicate_hash hash, array_references
+        array_references
+      end
+    end
+    
+    def deduplicate_hash hash, array_references
+      hash.each do |k, ary|
+        hash[k] = (array_references[ary] ||= ary)
+      end
+    end
+    
+  end
+end

data/lib/picky/optimizers.rb

@@ -0,0 +1,7 @@
+module Picky
+  module Optimizers
+    module Memory
+      
+    end
+  end
+end

data/lib/picky/query/tokens.rb

@@ -80,7 +80,7 @@ module Picky
         @tokens.map(&:original)
       end
       def original
-        @tokens.map(&:original)
+        originals
       end
       # TODO
       #

data/lib/picky.rb

@@ -11,10 +11,8 @@ module Picky
   # Set default encoding.
   # (Note: Rails does this already as well)
   #
-  # THINK Set default encoding?
-  #
-  # Encoding.default_external = Encoding::UTF_8
-  # Encoding.default_internal = Encoding::UTF_8
+  Encoding.default_external = Encoding::UTF_8
+  Encoding.default_internal = Encoding::UTF_8
 
   # External libraries.
   #

data/spec/functional/backends/file_spec.rb

@@ -7,7 +7,7 @@ require 'spec_helper'
 #
 describe Picky::Backends::File do
 
-  class Book
+  class PoetryBook
     attr_reader :id, :title, :author
     def initialize id, title, author
       @id, @title, @author = id, title, author
@@ -44,7 +44,7 @@ describe Picky::Backends::File do
       books.search('title').ids.should == []
     end
     it 'handles removing with more than one entry' do
-      data.add Book.new(2, 'title', 'author')
+      data.add PoetryBook.new(2, 'title', 'author')
 
       books.search('title').ids.should == ['2', '1']
 
@@ -53,8 +53,8 @@ describe Picky::Backends::File do
       books.search('title').ids.should == ['2']
     end
     it 'handles removing with three entries' do
-      data.add Book.new(2, 'title', 'author')
-      data.add Book.new(3, 'title', 'author')
+      data.add PoetryBook.new(2, 'title', 'author')
+      data.add PoetryBook.new(3, 'title', 'author')
 
       books.search('title').ids.should == ['3', '2', '1']
 
@@ -63,7 +63,7 @@ describe Picky::Backends::File do
       books.search('title').ids.should == ['3', '2']
     end
     it 'handles replacing' do
-      data.replace Book.new(1, 'toitle', 'oithor')
+      data.replace PoetryBook.new(1, 'toitle', 'oithor')
 
       books.search('title').ids.should == []
       books.search('toitle').ids.should == ['1']
@@ -100,7 +100,7 @@ describe Picky::Backends::File do
       books.search('title').ids.should == []
     end
     it 'handles removing with more than one entry' do
-      data.add Book.new(2, 'title', 'author')
+      data.add PoetryBook.new(2, 'title', 'author')
 
       books.search('title').ids.should == [2, 1]
 
@@ -109,8 +109,8 @@ describe Picky::Backends::File do
       books.search('title').ids.should == [2]
     end
     it 'handles removing with three entries' do
-      data.add Book.new(2, 'title', 'author')
-      data.add Book.new(3, 'title', 'author')
+      data.add PoetryBook.new(2, 'title', 'author')
+      data.add PoetryBook.new(3, 'title', 'author')
 
       books.search('title').ids.should == [3, 2, 1]
 
@@ -119,7 +119,7 @@ describe Picky::Backends::File do
       books.search('title').ids.should == [3, 2]
     end
     it 'handles replacing' do
-      data.replace Book.new(1, 'toitle', 'oithor')
+      data.replace PoetryBook.new(1, 'toitle', 'oithor')
 
       books.search('title').ids.should == []
       books.search('toitle').ids.should == [1]
@@ -144,7 +144,7 @@ describe Picky::Backends::File do
         data.backend described_class.new
         data.clear
 
-        data.add Book.new(1, 'title', 'author')
+        data.add PoetryBook.new(1, 'title', 'author')
       end
 
       instance_eval &its_to_s
@@ -157,7 +157,7 @@ describe Picky::Backends::File do
         data.backend described_class.new
         data.clear
 
-        data.add Book.new(1, 'title', 'author')
+        data.add PoetryBook.new(1, 'title', 'author')
       end
 
       instance_eval &its_to_i

data/spec/functional/no_dump_hint_spec.rb

@@ -5,7 +5,7 @@ require 'ostruct'
 
 describe "Hint: no_dump" do
 
-  Book = Struct.new(:id, :title, :author)
+  ComicBook = Struct.new(:id, :title, :author)
 
   let(:index) do
     Picky::Index.new :no_dump do
@@ -18,14 +18,14 @@ describe "Hint: no_dump" do
   let(:try) { Picky::Search.new index }
 
   it 'can index and search' do
-    index.replace Book.new(2, "Title", "Author")
+    index.replace ComicBook.new(2, "Title", "Author")
 
     try.search("title:title").ids.should == [2]
   end
   
   context 'dumping and loading' do
     it "raises" do
-      index.replace Book.new(2, "Title New", "Author New")
+      index.replace ComicBook.new(2, "Title New", "Author New")
 
       expect {
         index.dump

data/spec/functional/optimize_memory_spec.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+#
+require 'spec_helper'
+
+describe "Memory optimization" do
+  
+  it 'saves memory' do
+    # Remove all indexes.
+    Picky::Indexes.clear_indexes
+    
+    index = Picky::Index.new :memory_optimization do
+      category :text1
+      category :text2
+      category :text3
+      category :text4
+    end
+    try = Picky::Search.new index
+    
+    thing = Struct.new(:id, :text1, :text2, :text3, :text4)
+    
+    require 'objspace'
+    
+    GC.start
+    memsize_without_added_thing = ObjectSpace.memsize_of_all(Array)
+    GC.start
+    
+    index.add thing.new(1, 'one', 'two', 'three', 'four')
+    
+    GC.start
+    memsize_with_added_thing = ObjectSpace.memsize_of_all(Array)
+    GC.start
+    
+    memsize_with_added_thing.should > memsize_without_added_thing
+    
+    Picky::Indexes.optimize_memory
+    
+    GC.start
+    memsize_with_optimized_memory = ObjectSpace.memsize_of_all(Array)
+    GC.start
+    
+    memsize_with_optimized_memory.should < memsize_with_added_thing
+  end
+
+end

data/spec/lib/extensions/string_spec.rb

@@ -2,6 +2,9 @@
 #
 require 'spec_helper'
 
+# Requiring text here as it's optional in Picky.
+require 'text'
+
 describe String do
 
   context 'performance' do

data/spec/lib/indexes_indexing_spec.rb

@@ -45,7 +45,7 @@ describe Picky::Indexes do
       end
     end
     def self.it_forwards_each name
-      describe name do
+      describe "forwarding ##{name}" do
         it "calls #{name} on each in order" do
           @index1.should_receive(name).once.with(no_args).ordered
           @index2.should_receive(name).once.with(no_args).ordered

data/spec/lib/picky_spec.rb

@@ -2,17 +2,17 @@ require 'spec_helper'
 
 describe Picky do
 
-  it 'sets the right internal encoding' do
+  it 'sets the right external encoding' do
     Encoding.default_external.should == Encoding::UTF_8
   end
-  # THINK What to set default external encoding to?
+  # THINK What to set default internal encoding to?
   #
-  # it 'sets the right external encoding' do
-  #   Encoding.default_internal.should == Encoding::UTF_8
-  # end
+  it 'sets the right internal encoding' do
+    Encoding.default_internal.should == Encoding::UTF_8
+  end
   
   it 'loads in a simple ruby environment with the defined requirements' do
-    #TODO Picky.root is set to /spec/temp in spec_helper, so is this the "best" way?
+    # TODO Picky.root is set to /spec/temp in spec_helper, so is this the "best" way?
     load_path   = File.expand_path('../../../lib', __FILE__)
     ruby        = File.join(RbConfig::CONFIG['bindir'], RbConfig::CONFIG['ruby_install_name']).sub(/.*\s.*/m, '"\&"')
 

data/spec/lib/query/tokens_spec.rb

@@ -224,10 +224,24 @@ describe Picky::Query::Tokens do
     it 'should work correctly' do
       (@tokens + @other).to_s.should == 'Hello~ I~ Am A* Token~'
     end
+    it 'should work correctly' do
+      (@tokens + @other).texts.should == ['hello', 'i', 'am', 'a', 'token']
+    end
+    it 'should work correctly' do
+      (@tokens + @other).originals.should == ['Hello~', 'I~', 'Am', 'A*', 'Token~']
+    end
+    it 'correctly handles ignore_unassigned' do
+      (@tokens + @other).ignore_unassigned.should == false
+    end
+    it 'correctly handles ignore_unassigned' do
+      yet_another = described_class.new [], true
+      (@tokens + yet_another).ignore_unassigned.should == true
+      (yet_another + @tokens).ignore_unassigned.should == true
+    end
   end
 
   def self.it_should_forward name
-    describe name do
+    describe "forwarding ##{name}" do
       before(:each) do
         @internal_tokens = double :internal_tokens
         @tokens = described_class.new @internal_tokens

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: picky
 version: !ruby/object:Gem::Version
-  version: 4.27.1
+  version: 4.28.1
 platform: ruby
 authors:
 - Florian Hanke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-03 00:00:00.000000000 Z
+date: 2015-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,20 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2009.0'
-- !ruby/object:Gem::Dependency
-  name: google_hash
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
 description: Fast Ruby semantic text search engine with comfortable single field interface.
 email: florian.hanke+picky@gmail.com
 executables:
@@ -182,6 +168,8 @@ files:
 - lib/picky/loggers/default.rb
 - lib/picky/loggers/silent.rb
 - lib/picky/loggers/verbose.rb
+- lib/picky/optimizers.rb
+- lib/picky/optimizers/memory/array_deduplicator.rb
 - lib/picky/performant.rb
 - lib/picky/platforms/macruby.rb
 - lib/picky/pool.rb
@@ -255,6 +243,7 @@ files:
 - spec/functional/non_specific_ids_larger_than_20_spec.rb
 - spec/functional/object_use_spec.rb
 - spec/functional/only_spec.rb
+- spec/functional/optimize_memory_spec.rb
 - spec/functional/or_spec.rb
 - spec/functional/pool_spec.rb
 - spec/functional/range_queries_spec.rb
@@ -405,7 +394,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: http://rubyforge.org/projects/picky
-rubygems_version: 2.4.1
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: 'Picky: Semantic Search Engine. Clever Interface. Good Tools.'
@@ -439,6 +428,7 @@ test_files:
 - spec/functional/non_specific_ids_larger_than_20_spec.rb
 - spec/functional/object_use_spec.rb
 - spec/functional/only_spec.rb
+- spec/functional/optimize_memory_spec.rb
 - spec/functional/or_spec.rb
 - spec/functional/pool_spec.rb
 - spec/functional/range_queries_spec.rb