picky 3.1.11 → 3.1.12

data/lib/picky/backends/file/basic.rb

@@ -30,16 +30,26 @@ module Picky
           @mapping_file = Memory::JSON.new "#{cache_path}.file_mapping.#{extension}"
         end
 
+        # The default extension for index files is "index".
+        #
+        def extension
+          :index
+        end
+
         # The initial content before loading.
         #
-        def default
+        # Note: We could also load the mapping file
+        #       as in #load.
+        #
+        def initial
           nil
         end
 
-        # The default extension for index files is "index".
+        # The empty index that is used for putting the index
+        # together before it is saved into the files.
         #
-        def extension
-          :index
+        def empty
+          {}
         end
 
         # Will copy the index file to a location that

data/lib/picky/backends/file.rb

@@ -10,25 +10,25 @@ module Picky
     #
     class File < Backend
 
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
         JSON.new bundle.index_path(:inverted)
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         JSON.new bundle.index_path(:weights)
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         JSON.new bundle.index_path(:similarity)
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle

data/lib/picky/backends/memory/basic.rb

@@ -32,6 +32,13 @@ module Picky
           :index
         end
 
+        # The empty index that is used for putting the index
+        # together before it is dumped into the files.
+        #
+        def empty
+          {}
+        end
+
         # Will copy the index file to a location that
         # is in a directory named "backup" right under
         # the directory the index file is in.

data/lib/picky/backends/memory/json.rb

@@ -16,7 +16,7 @@ module Picky
 
         # The initial content before loading.
         #
-        def default
+        def initial
           {}
         end
 

data/lib/picky/backends/memory/marshal.rb

@@ -16,7 +16,7 @@ module Picky
 
         # The initial content before loading.
         #
-        def default
+        def initial
           {}
         end
 

data/lib/picky/backends/memory/text.rb

@@ -18,8 +18,8 @@ module Picky
 
         # The initial content before loading.
         #
-        def default
-          raise "Can't have a default content from text file. Use JSON or Marshal."
+        def initial
+          raise "Can't have an initial content from text file. Use JSON or Marshal."
         end
 
         # Text files are used exclusively for

data/lib/picky/backends/memory.rb

@@ -4,25 +4,25 @@ module Picky
 
     class Memory < Backend
 
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
         JSON.new bundle.index_path(:inverted)
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         JSON.new bundle.index_path(:weights)
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         Marshal.new bundle.index_path(:similarity)
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle

data/lib/picky/backends/redis/basic.rb

@@ -25,8 +25,11 @@ module Picky
 
         # The initial content before loading.
         #
-        def default
-          nil
+        # Note: As Redis indexes needn't be loaded per se,
+        #       this just returns the same thing as #load.
+        #
+        def initial
+          self
         end
 
         # Returns itself.
@@ -35,6 +38,13 @@ module Picky
           self
         end
 
+        # The empty index that is used for putting the index
+        # together.
+        #
+        def empty
+          {}
+        end
+
         # We do not use Redis to retrieve data.
         #
         def retrieve

data/lib/picky/backends/redis.rb

@@ -12,25 +12,25 @@ module Picky
         @client = options[:client] || ::Redis.new(:db => (options[:db] || 15))
       end
 
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => [id, id, id, id, id] (an array of ids)
       #
       def create_inverted bundle
         List.new client, "#{bundle.identifier}:inverted"
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:token] # => 1.23 (a weight)
       #
       def create_weights bundle
         Float.new client, "#{bundle.identifier}:weights"
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:encoded] # => [:original, :original] (an array of original symbols this similarity encoded thing maps to)
       #
       def create_similarity bundle
         List.new client, "#{bundle.identifier}:similarity"
       end
-      # Returns an object that responds to:
+      # Returns an object that on #initial, #load returns an object that responds to:
       #   [:key] # => value (a value for this config key)
       #
       def create_configuration bundle

data/lib/picky/bundle.rb

@@ -44,8 +44,9 @@ module Picky
     delegate :index_directory, :to => :category
 
     def initialize name, category, backend, weights_strategy, partial_strategy, similarity_strategy, options = {}
-      @name                = name
-      @category            = category
+      @name     = name
+      @category = category
+
       @weights_strategy    = weights_strategy
       @partial_strategy    = partial_strategy
       @similarity_strategy = similarity_strategy

data/lib/picky/categories_realtime.rb

@@ -3,25 +3,10 @@ module Picky
   class Categories
 
     each_delegate :remove,
+                  :add,
+                  :replace,
                   :to => :categories
 
-    # TODO Rewrite indexing/tokenizer/caching etc.
-    #
-    def add object
-      id = object.id
-      categories.each do |category|
-        tokens, _ = category.tokenizer.tokenize object.send(category.from).to_s
-        category.add id, tokens
-      end
-    end
-
-    # TODO
-    #
-    def replace object
-      remove object.id
-      add object
-    end
-
   end
 
 end

data/lib/picky/category_realtime.rb

@@ -1,25 +1,41 @@
 module Picky
 
-  # TODO
-  #
   class Category
 
-    # TODO
+    # Removes an indexed object with the
+    # given id.
     #
     def remove id
       indexed_exact.remove id
       indexed_partial.remove id
     end
 
-    # TODO
+    # Adds and indexes this category of the
+    # given object.
     #
-    def add id, tokens
+    def add object
+      tokens, _ = tokenizer.tokenize object.send(from).to_s
+      add_tokenized object.id, tokens
+    end
+
+    # Removes the object's id, and then
+    # adds it again.
+    #
+    def replace object
+      remove object.id
+      add object
+    end
+
+    # For the given id, adds the list of
+    # strings to the index for the given id.
+    #
+    def add_tokenized id, tokens
       tokens.each do |text|
         next unless text
         text = text.to_sym
         indexed_exact.add id, text
 
-        # TODO Refactor.
+        # TODO Refactor. Push into indexed_partial?
         #
         indexed_partial.partial_strategy.each_partial text do |partial_text|
           indexed_partial.add id, partial_text

data/lib/picky/indexed/bundle.rb

@@ -23,10 +23,10 @@ module Picky
       def initialize name, category, backend, weights_strategy, partial_strategy, similarity_strategy, options = {}
         super name, category, backend, weights_strategy, partial_strategy, similarity_strategy, options
 
-        @inverted      = @backend_inverted.default
-        @weights       = @backend_weights.default
-        @similarity    = @backend_similarity.default
-        @configuration = @backend_configuration.default
+        @inverted      = @backend_inverted.initial
+        @weights       = @backend_weights.initial
+        @similarity    = @backend_similarity.initial
+        @configuration = @backend_configuration.initial
 
         @realtime_mapping = {} # id -> ary of syms.  TODO Always instantiate?
       end

data/lib/picky/indexed/bundle_realtime.rb

@@ -21,7 +21,9 @@ module Picky
           if ids.empty?
             @inverted.delete   sym
             @weights.delete    sym
-            @similarity.delete encoded # Since no element uses this sym anymore, we can delete the similarity for it.
+            # Since no element uses this sym anymore, we can delete the similarity for it.
+            # TODO Not really. Since multiple syms can point to the same encoded.
+            @similarity.delete encoded
           else
             @weights[sym] = self.weights_strategy.weight_for ids.size
           end

data/lib/picky/indexing/bundle.rb

@@ -43,10 +43,10 @@ module Picky
         @key_format       = options[:key_format]
         @prepared         = Backends::Memory::Text.new category.prepared_index_path
 
-        @inverted      = {}
-        @weights       = {}
-        @similarity    = {}
-        @configuration = {}
+        @inverted      = @backend_inverted.empty
+        @weights       = @backend_weights.empty
+        @similarity    = @backend_similarity.empty
+        @configuration = @backend_configuration.empty
       end
 
       # Sets up a piece of the index for the given token.

data/spec/lib/backends/memory/basic_spec.rb

@@ -2,23 +2,29 @@ require 'spec_helper'
 
 describe Picky::Backends::Memory::Basic do
   
-  let(:file) { described_class.new 'some/cache/path/to/file' }
-  
+  let(:basic) { described_class.new 'some/cache/path/to/file' }
+
+  describe 'empty' do
+    it 'returns the container that is used for indexing' do
+      basic.empty.should == {}
+    end
+  end
+
   describe 'backup_file_path_of' do
     it 'returns a backup path relative to the path' do
-      file.backup_file_path_of('some/path/to/some.memory.index').should == 'some/path/to/backup/some.memory.index'
+      basic.backup_file_path_of('some/path/to/some.memory.index').should == 'some/path/to/backup/some.memory.index'
     end
   end
   
   describe 'backup_directory' do
     it "returns the cache path's backup path" do
-      file.backup_directory('some/cache/path/to/file').should == 'some/cache/path/to/backup'
+      basic.backup_directory('some/cache/path/to/file').should == 'some/cache/path/to/backup'
     end
   end
   
   describe 'to_s' do
     it 'returns the cache path with the default file extension' do
-      file.to_s.should == 'Picky::Backends::Memory::Basic(some/cache/path/to/file.memory.index)'
+      basic.to_s.should == 'Picky::Backends::Memory::Basic(some/cache/path/to/file.memory.index)'
     end
   end
   

data/spec/lib/backends/memory/json_spec.rb

@@ -2,29 +2,43 @@ require 'spec_helper'
 
 describe Picky::Backends::Memory::JSON do
   
-  let(:file) { described_class.new 'some/cache/path/to/file' }
-  
-  describe "dump" do
-    it "delegates to the given hash" do
-      hash = stub :hash
-      
-      hash.should_receive(:dump_json).once.with "some/cache/path/to/file.memory.json"
-      
-      file.dump hash
+  context 'hash-based indexes' do
+    let(:json) { described_class.new 'some/cache/path/to/file' }
+
+    describe 'extension' do
+      it 'is correct' do
+        json.extension.should == :json
+      end
     end
-  end
-  
-  describe "retrieve" do
-    it "raises" do
-      lambda do
-        file.retrieve
-      end.should raise_error("Can't retrieve from JSON file. Use text file.")
+
+    describe 'initial' do
+      it 'is correct' do
+        json.initial.should == {}
+      end
     end
-  end
-  
-  describe 'to_s' do
-    it 'returns the cache path with the default file extension' do
-      file.to_s.should == 'Picky::Backends::Memory::JSON(some/cache/path/to/file.memory.json)'
+
+    describe "dump" do
+      it "delegates to the given hash" do
+        hash = stub :hash
+
+        hash.should_receive(:dump_json).once.with "some/cache/path/to/file.memory.json"
+
+        json.dump hash
+      end
+    end
+
+    describe "retrieve" do
+      it "raises" do
+        lambda do
+          json.retrieve
+        end.should raise_error("Can't retrieve from JSON file. Use text file.")
+      end
+    end
+
+    describe 'to_s' do
+      it 'returns the cache path with the default file extension' do
+        json.to_s.should == 'Picky::Backends::Memory::JSON(some/cache/path/to/file.memory.json)'
+      end
     end
   end
   

data/spec/lib/backends/memory/marshal_spec.rb

@@ -2,29 +2,43 @@ require 'spec_helper'
 
 describe Picky::Backends::Memory::Marshal do
   
-  let(:file) { described_class.new 'some/cache/path/to/file' }
+  context 'hash-based indexes' do
+    let(:marshal) { described_class.new 'some/cache/path/to/file' }
   
-  describe "dump" do
-    it "delegates to the given hash" do
-      hash = stub :hash
+    describe 'extension' do
+      it 'is correct' do
+        marshal.extension.should == :dump
+      end
+    end
+
+    describe 'initial' do
+      it 'is correct' do
+        marshal.initial.should == {}
+      end
+    end
+    
+    describe "dump" do
+      it "delegates to the given hash" do
+        hash = stub :hash
       
-      hash.should_receive(:dump_marshal).once.with "some/cache/path/to/file.memory.dump"
+        hash.should_receive(:dump_marshal).once.with "some/cache/path/to/file.memory.dump"
       
-      file.dump hash
+        marshal.dump hash
+      end
     end
-  end
   
-  describe "retrieve" do
-    it "raises" do
-      lambda do
-        file.retrieve
-      end.should raise_error("Can't retrieve from marshalled file. Use text file.")
+    describe "retrieve" do
+      it "raises" do
+        lambda do
+          marshal.retrieve
+        end.should raise_error("Can't retrieve from marshalled file. Use text file.")
+      end
     end
-  end
   
-  describe 'to_s' do
-    it 'returns the cache path with the default file extension' do
-      file.to_s.should == 'Picky::Backends::Memory::Marshal(some/cache/path/to/file.memory.dump)'
+    describe 'to_s' do
+      it 'returns the cache path with the default file extension' do
+        marshal.to_s.should == 'Picky::Backends::Memory::Marshal(some/cache/path/to/file.memory.dump)'
+      end
     end
   end
   

data/spec/lib/backends/memory/text_spec.rb

@@ -2,21 +2,33 @@ require 'spec_helper'
 
 describe Picky::Backends::Memory::Text do
   
-  before(:each) do
-    @file = described_class.new "some_cache_path"
+  let(:text) { described_class.new "some_cache_path" }
+  
+  describe 'extension' do
+    it 'is correct' do
+      text.extension.should == :txt
+    end
+  end
+
+  describe 'initial' do
+    it 'raises' do
+      expect {
+        text.initial
+      }.to raise_error("Can't have an initial content from text file. Use JSON or Marshal.")
+    end
   end
   
   describe "load" do
     it "raises" do
       lambda do
-        @file.load
+        text.load
       end.should raise_error("Can't load from text file. Use JSON or Marshal.")
     end
   end
   describe "dump" do
     it "raises" do
       lambda do
-        @file.dump :anything
+        text.dump :anything
       end.should raise_error("Can't dump to text file. Use JSON or Marshal.")
     end
   end
@@ -27,13 +39,13 @@ describe Picky::Backends::Memory::Text do
       ::File.should_receive(:open).any_number_of_times.and_yield @io
     end
     it "yields split lines and returns the id and token text" do
-      @file.retrieve do |id, token|
+      text.retrieve do |id, token|
         id.should    == '123456'
         token.should == :some_nice_token
       end
     end
     it "is fast" do
-      performance_of { @file.retrieve { |id, token| } }.should < 0.00005
+      performance_of { text.retrieve { |id, token| } }.should < 0.00005
     end
   end
   

data/spec/lib/backends/redis/basic_spec.rb

@@ -16,6 +16,18 @@ describe Picky::Backends::Redis::Basic do
     end
   end
   
+  describe 'empty' do
+    it 'returns the container that is used for indexing' do
+      index.empty.should == {}
+    end
+  end
+  
+  describe 'initial' do
+    it 'is correct' do
+      index.initial.class.should == described_class
+    end
+  end
+  
   describe 'cache_small?' do
     context 'size 0' do
       before(:each) do

data/spec/specific/realtime_spec.rb

@@ -12,7 +12,7 @@ describe "Realtime Indexing" do
   end
   
   let(:index) do
-    Picky::Index.new(:test) do
+    Picky::Index.new(:books) do
       source []
       category :title
       category :author, similarity: Picky::Generators::Similarity::DoubleMetaphone.new(3)
@@ -24,6 +24,39 @@ describe "Realtime Indexing" do
     index.add Book.new(1, "Title", "Author")
   end
   
+  context 'single category updating' do
+    it 'finds the first entry' do
+      books.search('title:Titl').ids.should == [1]
+    end
+    
+    it 'allows removing a single category and leaving the others alone' do
+      index[:title].remove 1
+      
+      books.search('Title').ids.should == []
+      books.search('Author').ids.should == [1]
+    end
+    
+    it 'allows adding a single category and leaving the others alone' do
+      index[:title].add Book.new(2, "Newtitle", "Newauthor")
+      
+      books.search('Title').ids.should == [1]
+      books.search('Newtitle').ids.should == [2]
+      
+      books.search('Author').ids.should == [1]
+      books.search('Newauthor').ids.should == []
+    end
+    
+    it 'allows replacing a single category and leaving the others alone' do
+      index[:title].replace Book.new(1, "Replaced", "Notindexed")
+      
+      books.search('Title').ids.should == []
+      books.search('Replaced').ids.should == [1]
+      
+      books.search('Notindexed').ids.should == []
+      books.search('Author').ids.should == [1]
+    end
+  end
+  
   context 'with partial' do
     it 'finds the first entry' do
       books.search('Titl').ids.should == [1]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: picky
 version: !ruby/object:Gem::Version
-  version: 3.1.11
+  version: 3.1.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-24 00:00:00.000000000 +11:00
-default_executable: picky
+date: 2011-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70264818405600 !ruby/object:Gem::Requirement
+  requirement: &70305699965140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,18 +21,18 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70264818405600
+  version_requirements: *70305699965140
 - !ruby/object:Gem::Dependency
   name: picky-client
-  requirement: &70264818404920 !ruby/object:Gem::Requirement
+  requirement: &70305699964660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
       - !ruby/object:Gem::Version
-        version: 3.1.11
+        version: 3.1.12
   type: :development
   prerelease: false
-  version_requirements: *70264818404920
+  version_requirements: *70305699964660
 description: Fast Ruby semantic text search engine with comfortable single field interface.
 email: florian.hanke+picky@gmail.com
 executables:
@@ -269,7 +268,6 @@ files:
 - spec/specific/realtime_spec.rb
 - spec/specific/speed_spec.rb
 - bin/picky
-has_rdoc: true
 homepage: http://florianhanke.com/picky
 licenses: []
 post_install_message: 
@@ -290,7 +288,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: http://rubyforge.org/projects/picky
-rubygems_version: 1.6.2
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: ! 'Picky: Semantic Search Engine. Clever Interface. Good Tools.'