acts_as_hashish 0.2.0 → 0.3.0

data/lib/acts_as_hashish.rb

@@ -1,5 +1,6 @@
 require 'redis'
 require 'json'
+require 'timeout'
 require 'acts_as_hashish'
 require 'acts_as_hashish/version'
 require 'acts_as_hashish/configuration'

data/lib/acts_as_hashish/hashish.rb

@@ -19,77 +19,105 @@ module Hashish
       key_value = key.is_a?(Proc) ? key.call(item) : item[key]
     end
 
+    def remove_hashish_metadata(key)
+      prefix = @options[:key_prefix]
+      Hashish.redis_connection.smembers("#{prefix}#INDEXES").each do |member|
+        Hashish.redis_connection.zrem(member, "#{prefix}@#{key}")
+        Hashish.redis_connection.srem("#{prefix}#INDEXES", member) if Hashish.redis_connection.zcard(member) == 0
+      end
+      Hashish.redis_connection.smembers("#{prefix}#SORTERS").each do |member|
+        if member =~ /^#{prefix}\$[\w]+@#{key}$/
+          Hashish.redis_connection.del(member)
+          Hashish.redis_connection.srem("#{prefix}#SORTERS", member)
+        end
+       end
+    end
+
+    def add_hashish_metadata(name, category)
+      prefix = @options[:key_prefix]
+      Hashish.redis_connection.sadd("#{prefix}##{category}", name)
+    end
+
+    def redis_keys
+      Hashish.redis_connection.keys("#{@options[:key_prefix]}*").inject({}){|h,x| h[x] = Hashish.redis_connection.type(x);h}
+    end
+
+    def wait_on_lock(timeout = 0)
+      lock_key = "#{@options[:key_prefix]}#MUTEX_LOCK"
+      begin
+        Timeout::timeout(timeout) do 
+          while Hashish.redis_connection.incr(lock_key) != 1
+            sleep(1)
+          end
+        end
+        yield if block_given?
+      ensure
+        Hashish.redis_connection.del(lock_key)
+      end
+    end
+
     public
-    def rebuild_indexes
-      get_list(:page_size => 0).each do |item|
+    def hashish_rebuild
+      temp = nil
+      wait_on_lock do
+        temp = hashish_list(:page_size => 0)
+        hashish_flush!
+      end
+      yield if block_given?
+      temp.each do |item|
         hashish_insert(item)
       end
     end
-    
-    def flush!
+
+    def hashish_flush!
       Hashish.redis_connection.keys("#{@options[:key_prefix]}*").each{|x| Hashish.redis_connection.del(x)}
     end
 
-    def size(category = '')
-      Hashish.redis_connection.zcard("#{@options[:key_prefix]}:#{category}")
+    def hashish_length
+      Hashish.redis_connection.zcard("#{@options[:key_prefix]}*")
     end
     
     def hashish_delete(key = nil)
-      key ||= get_hashish_key(get_hashish_list.first)
+      key ||= get_hashish_key(hashish_list.first)
       prefix = @options[:key_prefix]
-      Hashish.redis_connection.smembers("#{prefix}:status:*").each do |v|
-        Hashish.redis_connection.zrem(v, "#{prefix}:#{key}")
-      end
-      Hashish.redis_connection.del("#{prefix}:#{key}")
+      Hashish.redis_connection.zrem("#{prefix}*", "#{prefix}@#{key}")
+      remove_hashish_metadata("#{key}")
+      Hashish.redis_connection.del("#{prefix}@#{key}")
     end
     
-    def hashish_insert(o, category = '', t = Time.now.to_i)
-      data = o.to_json
-      prefix = @options[:key_prefix]
-      category += ':' unless category.empty?
-      key = get_hashish_key(o)
-      
-      Hashish.redis_connection.zadd("#{prefix}:#{category}", t , "#{prefix}:#{key}")
-      Hashish.redis_connection.zremrangebyrank("#{prefix}:#{category}", 0, -(@options[:max_size] + 1)) if @options[:max_size]
-      Hashish.redis_connection.sadd("#{prefix}:status:*", "#{prefix}:#{category}")
-      @options[:indexes].each do |index_field, index|
-        index_value = index.is_a?(Proc) ? index.call(o) : o[index]
-        if index_value.is_a?(Array)
-          index_value.each do |i|
-            Hashish.redis_connection.zadd("#{prefix}:#{index_field}:#{i}", t, "#{prefix}:#{key}") if i.is_a?(String)
+    def hashish_insert(o, t = Time.now.to_i)
+      wait_on_lock do 
+        data = o.to_json
+        prefix = @options[:key_prefix]
+        key = get_hashish_key(o)
+        raise "Error: Computed data key as '#{key}'. Only alphanumeric characters allowed!" unless key.to_s =~ /^[\w]+$/
+        Hashish.redis_connection.multi do 
+          Hashish.redis_connection.zadd("#{prefix}*", t , "#{prefix}@#{key}")
+          # Hashish.redis_connection.zremrangebyrank("#{prefix}:", 0, -(@options[:max_size] + 1)) if @options[:max_size]
+          @options[:indexes].each do |index_field, index|
+            index_value = index.is_a?(Proc) ? index.call(o) : o[index]
+            if index_value.is_a?(Array)
+              index_value.each do |i|
+                Hashish.redis_connection.zadd("#{prefix}!#{index_field}=#{i}", t, "#{prefix}@#{key}") if i.is_a?(String)
+                add_hashish_metadata("#{prefix}!#{index_field}=#{i}", 'INDEXES')
+              end
+            else
+              Hashish.redis_connection.zadd("#{prefix}!#{index_field}=#{index_value}", t, "#{prefix}@#{key}")
+              add_hashish_metadata("#{prefix}!#{index_field}=#{index_value}", 'INDEXES')
+            end
           end
-        else
-          Hashish.redis_connection.zadd("#{prefix}:#{index_field}:#{index_value}", t, "#{prefix}:#{key}")
+          @options[:sorters].each do |sort_field, sort|
+            sort_value = sort.is_a?(Proc) ? sort.call(o) : o[sort]
+            Hashish.redis_connection.set("#{prefix}$#{sort_field}@#{key}", sort_value)
+            add_hashish_metadata("#{prefix}$#{sort_field}@#{key}", 'SORTERS')
+          end
+          Hashish.redis_connection.set("#{prefix}@#{key}", data)
         end
       end
-      @options[:sorters].each do |sort_field, sort|
-        sort_value = sort.is_a?(Proc) ? sort.call(o) : o[sort]
-        Hashish.redis_connection.set("#{prefix}:#{key}:#{sort_field}", sort_value)
-      end
-      Hashish.redis_connection.set("#{prefix}:#{key}", data)
       true
     end
 
-    def hashish_list(*args)
-      category = nil
-      
-      options = {}
-      
-      # handle all kinds of argument structures that make sense
-      if args.length == 1
-        if args[0].is_a?(Hash)
-          options = args[0]
-        elsif args[0].is_a?(String)
-          category = args[0]
-        else
-          raise
-        end
-      elsif args.length > 1
-        category = args[0]
-        options = args[1]
-      end
-      
-      category += ':' if category          
+    def hashish_list(options = {})
       page_no = options[:page_no] || 1
       page_size = options[:page_size] || 10
       
@@ -104,7 +132,7 @@ module Hashish
       limit = offset + page_size
       
       # get the next seq no for search operations for this search instance (perhaps this entire section shld be oops based stuff but what the heck :E mayb later)
-      seq = Hashish.redis_connection.incr("#{@options[:key_prefix]}:result:seq")
+      seq = Hashish.redis_connection.incr("#{@options[:key_prefix]}#SEQUENCER")
       
       # search
       inter = []
@@ -114,28 +142,28 @@ module Hashish
         filters.each do |key, value|
           if value.is_a?(Array)
             union = []
-            union_key = "#{@options[:key_prefix]}:union:#{key}:#{seq}"
+            union_key = "#{@options[:key_prefix]}#UNION##{key}##{seq}"
             value.each do |v|
-              union << "#{@options[:key_prefix]}:#{key}:#{v}"
+              union << "#{@options[:key_prefix]}!#{key}=#{v}"
             end
             Hashish.redis_connection.zunionstore(union_key, union.uniq)
             Hashish.redis_connection.expire(union_key, Hashish.redis_search_keys_ttl)
             inter << union_key
           else
-            inter << "#{@options[:key_prefix]}:#{key}:#{value}"
+            inter << "#{@options[:key_prefix]}!#{key}=#{value}"
           end
         end
         
       end
       
-      full_list = "#{@options[:key_prefix]}:#{category}"
+      full_list = "#{@options[:key_prefix]}*"
       
       # is the user askin for a cropped set of data (min/max date/time of nQ)
       if min_time == '-inf' and max_time == '+inf'
         inter << full_list
       else
         # copy the full list to different temp set
-        all_items_key = "#{@options[:key_prefix]}:all_items:#{seq}"
+        all_items_key = "#{@options[:key_prefix]}#ALL_ITEMS##{seq}"
         Hashish.redis_connection.zunionstore(all_items_key, [full_list])
         Hashish.redis_connection.expire(all_items_key, Hashish.redis_search_keys_ttl * 60)
         # crop the set based on min/max (wish we had a 1 step zcroprangebyscore)
@@ -144,15 +172,15 @@ module Hashish
         inter << all_items_key
       end
       
-      result_key = "#{@options[:key_prefix]}:result:#{seq}"
+      result_key = "#{@options[:key_prefix]}#RESULT##{seq}"
       Hashish.redis_connection.zinterstore(result_key, inter, :aggregate => 'max')
       Hashish.redis_connection.expire(result_key, Hashish.redis_search_keys_ttl * 60)
       result = nil
       if sort_by
-        custom_sort = "#{@options[:key_prefix]}:custom_sort:#{seq}"
-        Hashish.redis_connection.sort(result_key, :by => "*:#{sort_by}",:get => '*', :store => custom_sort, :order => sort_order)
-        Hashish.redis_connection.expire(custom_sort, Hashish.redis_search_keys_ttl * 60)
-        result = Hashish.redis_connection.lrange(custom_sort, offset, limit -1)
+        custom_sort_key = "#{@options[:key_prefix]}#CUSTOM_SORT##{seq}"
+        Hashish.redis_connection.sort(result_key, :by => "*:#{sort_by}",:get => '*', :store => custom_sort_key, :order => sort_order)
+        Hashish.redis_connection.expire(custom_sort_key, Hashish.redis_search_keys_ttl * 60)
+        result = Hashish.redis_connection.lrange(custom_sort_key, offset, limit -1)
       else
         res_keys = Hashish.redis_connection.send("z#{(sort_order == 'DESC' ? 'rev' : '')}range".to_sym, result_key, offset, limit - 1)
         if res_keys.empty?

data/lib/acts_as_hashish/version.rb

@@ -1,3 +1,3 @@
 module Hashish
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/spec/hashish_spec.rb

@@ -1,14 +1,123 @@
 require 'spec_helper'
 
 describe Hashish do
+
+  before :each do
+    @sample_data_1 = {'id' => 1, 'name' => 'Joe'}
+    @sample_data_2 = {'id' => 3, 'name' => 'Doe'}
+    @sample_data_3 = {'id' => 2, 'name' => 'Moe'}
+    @sample_data_4 = {'id' => 4, 'name' => 'Loe'}
+
+    @primary_key = 'id'
+
+    @index_name = '_name'
+    @index_value = 'name'
+
+    @proc_index_name = 'id_name'
+    @proc_index_value = Proc.new{|x| "#{x['id']}=>#{x['name']}"}
+
+    @sorter_name = '_id'
+    @sorter_value = 'id'
+
+    SampleClass.acts_as_hashish(:key => @primary_key, :indexes => {@index_name => @index_value, @proc_index_name => @proc_index_value}, :sorters => {@sorter_name => @sorter_value})
+
+    @options = SampleClass.instance_variable_get(:@options)
+  end
   
   describe ".acts_as_hashish" do
     it "should hashify the class" do
-      SampleClass.expects(:hashify)
-      SampleClass.acts_as_hashish(:key => 'id')
-      SampleClass.instance_variable_get(:@options)[:key_prefix].should eq(Hashish.redis_namespace + ":" + SampleClass.to_s)
-      puts SampleClass.methods.sort
+      @options = SampleClass.instance_variable_get(:@options)
+      @options[:key_prefix].should eq(Hashish.redis_namespace + ":" + SampleClass.to_s)
+      [:hashish_rebuild, :hashish_flush!, :hashish_length, :hashish_insert, :hashish_delete, :hashish_list].each do |m|
+        SampleClass.public_methods.include?(m).should be_true
+      end
+    end
+  end
+
+  describe ".hashish_insert" do
+    it "should insert the data into the list" do
+      SampleClass.hashish_insert(@sample_data_1)
+      Hashish.redis_connection.zscore("#{@options[:key_prefix]}*", "#{@options[:key_prefix]}@#{SampleClass.send(:get_hashish_key, @sample_data_1)}").should_not be_nil
+    end
+
+    it "should create appropriate indexes" do
+      SampleClass.hashish_insert(@sample_data_1)
+      Hashish.redis_connection.zscore("#{@options[:key_prefix]}!#{@index_name}=#{@sample_data_1[@index_value]}", "#{@options[:key_prefix]}@#{SampleClass.send(:get_hashish_key, @sample_data_1)}").should_not be_nil
+      Hashish.redis_connection.zscore("#{@options[:key_prefix]}!#{@proc_index_name}=#{@proc_index_value.call(@sample_data_1)}", "#{@options[:key_prefix]}@#{SampleClass.send(:get_hashish_key, @sample_data_1)}").should_not be_nil
+    end
+
+    it "should create appropriate sorters" do
+      SampleClass.hashish_insert(@sample_data_1)
+      Hashish.redis_connection.get("#{@options[:key_prefix]}$#{@sorter_name}@#{SampleClass.send(:get_hashish_key, @sample_data_1)}").should == @sample_data_1[@sorter_value].to_s
     end
   end
+
+  describe ".hashish_delete" do
+    it "should delete the data from the list" do
+      SampleClass.hashish_insert(@sample_data_1)
+      SampleClass.hashish_delete(@sample_data_1[@primary_key])
+      Hashish.redis_connection.zscore("#{@options[:key_prefix]}:", "#{@options[:key_prefix]}:#{SampleClass.send(:get_hashish_key, @sample_data_1)}").should be_nil
+      Hashish.redis_connection.exists("#{@options[:key_prefix]}:#{SampleClass.send(:get_hashish_key, @sample_data_1)}").should be_false
+    end
+
+    it "should delete appropriate sorters and indexes" do
+      SampleClass.hashish_insert(@sample_data_1)
+      SampleClass.hashish_delete(@sample_data_1[@primary_key])
+      SampleClass.send(:redis_keys).should == {}
+    end
+  end
+
+  describe ".hashish_list" do
+    it "should list the items from the list"
+  end
+
+  describe ".hashish_flush!" do
+    it "should delete all keys pertaining to this class" do
+      SampleClass.hashish_insert(@sample_data_1)
+      SampleClass.hashish_insert(@sample_data_2)
+      SampleClass.hashish_insert(@sample_data_3)
+      SampleClass.hashish_flush!
+      SampleClass.send(:redis_keys).should == {}
+    end
+  end
+
+  describe ".hashish_length" do
+    it "should return the length of the list" do
+      SampleClass.hashish_insert(@sample_data_1)
+      SampleClass.hashish_insert(@sample_data_2)
+      SampleClass.hashish_length.should == 2
+    end
+  end
+
+  describe ".hashish_rebuild" do
+    it "should rebuild the list correctly" do
+      SampleClass.hashish_insert(@sample_data_1)
+      SampleClass.hashish_insert(@sample_data_2)
+      SampleClass.hashish_insert(@sample_data_3)
+      before = SampleClass.send(:redis_keys) 
+      SampleClass.hashish_rebuild
+      after = SampleClass.send(:redis_keys) 
+      before.should == after
+    end
+  end
+  
+  describe ".wait_on_lock" do
+    it "should avoid concurrency issues" do
+      x = 0        
+      threads = []
+      (0..1).each do |i|
+        threads << Thread.new do
+          SampleClass.send(:wait_on_lock) do
+            a = x + 1
+            sleep(5)
+            x = a
+          end
+        end
+      end
+      threads.each{|t|t.join}
+      x.should == 2
+    end
+  end
+  
 end
 

metadata

@@ -2,7 +2,7 @@
 name: acts_as_hashish
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Schubert Cardozo