rhosync 2.0.0.beta1

data/spec/source_sync_spec.rb

@@ -0,0 +1,248 @@
+require File.join(File.dirname(__FILE__),'spec_helper')
+
+describe "SourceSync" do
+  it_should_behave_like "SpecBootstrapHelper"
+  it_should_behave_like "SourceAdapterHelper"
+  
+  before(:each) do
+    @ss = SourceSync.new(@s)
+  end
+  
+  it "should create SourceSync" do
+    @ss.adapter.is_a?(SampleAdapter).should == true
+  end
+  
+  it "should fail to create SourceSync with InvalidArgumentError" do
+    lambda { SourceSync.new(nil) }.should raise_error(InvalidArgumentError, 'Invalid source')
+  end
+  
+  it "should raise SourceAdapterLoginException if login fails" do
+    msg = "Error logging in"
+    @u.login = nil
+    @ss = SourceSync.new(@s)
+    @ss.should_receive(:log).with("SourceAdapter raised login exception: #{msg}")
+    @ss.process_query
+    verify_result(@s.docname(:errors) => {'login-error'=>{'message'=>msg}})
+  end
+  
+  it "should raise SourceAdapterLogoffException if logoff fails" do
+    msg = "Error logging off"
+    @ss.should_receive(:log).with("SourceAdapter raised logoff exception: #{msg}")
+    set_test_data('test_db_storage',{},msg,'logoff error')
+    @ss.process_query
+    verify_result(@s.docname(:errors) => {'logoff-error'=>{'message'=>msg}})
+  end
+  
+  it "should hold on read on subsequent call of process" do
+    expected = {'1'=>@product1}
+    Store.put_data('test_db_storage',expected)
+    @ss.process_query
+    Store.put_data('test_db_storage',{'2'=>@product2})
+    @ss.process_query
+    verify_result(@s.docname(:md) => expected)   
+  end
+  
+  it "should read on every subsequent call of process" do
+    expected = {'2'=>@product2}
+    @s.poll_interval = 0
+    Store.put_data('test_db_storage',{'1'=>@product1})
+    @ss.process_query
+    Store.put_data('test_db_storage',expected)
+    @ss.process_query
+    verify_result(@s.docname(:md) => expected)    
+  end
+
+  it "should never call read on any call of process" do
+    @s.poll_interval = -1
+    Store.put_data('test_db_storage',{'1'=>@product1})
+    @ss.process_query
+    verify_result(@s.docname(:md) => {})
+  end
+    
+  describe "methods" do
+    
+    it "should process source adapter" do
+      mock_metadata_method([SampleAdapter, SimpleAdapter]) do
+        expected = {'1'=>@product1,'2'=>@product2}
+        set_state('test_db_storage' => expected)
+        @ss.process_query
+        verify_result(@s.docname(:md) => expected,
+          @s.docname(:metadata) => "{\"foo\":\"bar\"}",
+          @s.docname(:metadata_sha1) => "a5e744d0164540d33b1d7ea616c28f2fa97e754a")
+      end
+    end
+    
+    it "should call methods in source adapter" do
+      mock_metadata_method([SampleAdapter, SimpleAdapter]) do
+        expected = {'1'=>@product1,'2'=>@product2}
+        metadata = "{\"foo\":\"bar\"}"
+        @ss.adapter.should_receive(:login).once.with(no_args()).and_return(true)
+        @ss.adapter.should_receive(:metadata).once.with(no_args()).and_return(metadata)
+        @ss.adapter.should_receive(:query).once.with(no_args()).and_return(expected)
+        @ss.adapter.should_receive(:sync).once.with(no_args()).and_return(true)
+        @ss.adapter.should_receive(:logoff).once.with(no_args()).and_return(nil)
+        @ss.process_query
+      end
+    end
+    
+    describe "create" do
+      it "should do create where adapter.create returns nil" do
+        set_state(@c.docname(:create) => {'2'=>@product2})
+        @ss.create(@c.id)
+        verify_result(@c.docname(:create_errors) => {},
+          @c.docname(:create_links) => {},
+          @c.docname(:create) => {})
+      end
+    
+      it "should do create where adapter.create returns object link" do
+        @product4['link'] = 'test link'
+        set_state(@c.docname(:create) => {'4'=>@product4})
+        @ss.create(@c.id)
+        verify_result(@c.docname(:create_errors) => {},
+          @c.docname(:create_links) => {'4'=>{'l'=>'backend_id'}},
+          @c.docname(:create) => {})
+      end
+    
+      it "should raise exception on adapter.create" do
+        msg = "Error creating record"
+        data = add_error_object({'4'=>@product4,'2'=>@product2},msg)
+        set_state(@c.docname(:create) => data)
+        @ss.create(@c.id)
+        verify_result(@c.docname(:create_errors) => 
+          {"#{ERROR}-error"=>{"message"=>msg},ERROR=>data[ERROR]})
+      end
+    end
+    
+    describe "update" do
+      it "should do update with no errors" do
+        set_state(@c.docname(:update) => {'4'=> { 'price' => '199.99' }})
+        @ss.update(@c.id)
+        verify_result(@c.docname(:update_errors) => {},
+          @c.docname(:update) => {})
+      end
+      
+      it "should do update with errors" do
+        msg = "Error updating record"
+        data = add_error_object({'4'=> { 'price' => '199.99' }},msg)
+        set_state(@c.docname(:update) => data)
+        @ss.update(@c.id)
+        verify_result(@c.docname(:update_errors) =>
+          {"#{ERROR}-error"=>{"message"=>msg}, ERROR=>data[ERROR]},
+            @c.docname(:update) => {'4'=> { 'price' => '199.99'}})
+      end
+    end
+    
+    describe "delete" do
+      it "should do delete with no errors" do
+        set_state(@c.docname(:delete) => {'4'=>@product4},
+          @s.docname(:md) => {'4'=>@product4,'3'=>@product3},
+          @c.docname(:cd) => {'4'=>@product4,'3'=>@product3})
+        @ss.delete(@c.id)
+        verify_result(@c.docname(:delete_errors) => {},
+          @s.docname(:md) => {'3'=>@product3},
+          @c.docname(:cd) => {'3'=>@product3},
+          @c.docname(:delete) => {})
+      end
+      
+      it "should do delete with errors" do
+        msg = "Error delete record"
+        data = add_error_object({'2'=>@product2},msg)
+        set_state(@c.docname(:delete) => data)
+        @ss.delete(@c.id)
+        verify_result(@c.docname(:delete_errors) => 
+          {"#{ERROR}-error"=>{"message"=>msg}, ERROR=>data[ERROR]},
+            @c.docname(:delete) => {'2'=>@product2})
+      end
+    end
+    
+    describe "cud" do
+      it "should do process_cud" do
+        @ss.should_receive(:_auth_op).twice.and_return(true)
+        @ss.should_receive(:create).once.with(@c.id)
+        @ss.should_receive(:update).once.with(@c.id)
+        @ss.should_receive(:delete).once.with(@c.id)
+        @ss.process_cud(@c.id)
+      end
+    end
+    
+    describe "query" do
+      it "should do query with no exception" do
+        verify_read_operation('query')
+      end
+      
+      it "should do query with exception raised" do
+        verify_read_operation_with_error('query')
+      end
+    end
+    
+    describe "search" do
+      it "should do search with no exception" do
+        verify_read_operation('search')
+      end
+      
+      it "should do search with exception raised" do
+        verify_read_operation_with_error('search')
+      end
+    end
+    
+    describe "app-level partitioning" do
+      it "should create app-level masterdoc with '__shared__' docname" do
+        @s1 = Source.load(@s_fields[:name],@s_params)
+        @s1.partition = :app
+        @ss1 = SourceSync.new(@s1)
+        expected = {'1'=>@product1,'2'=>@product2}
+        set_state('test_db_storage' => expected)
+        @ss1.process_query
+        verify_result("source:#{@test_app_name}:__shared__:#{@s_fields[:name]}:md" => expected)
+        Store.db.keys("read_state:#{@test_app_name}:__shared__*").sort.should ==
+          [ "read_state:#{@test_app_name}:__shared__:SampleAdapter:refresh_time",
+            "read_state:#{@test_app_name}:__shared__:SampleAdapter:rho__id"]
+      end
+    end
+    
+    def verify_read_operation(operation)
+      expected = {'1'=>@product1,'2'=>@product2}
+      set_test_data('test_db_storage',expected)
+      Store.put_data(@s.docname(:errors),
+        {"#{operation}-error"=>{'message'=>'failed'}},true)
+      if operation == 'query'
+        @ss.read.should == true
+        verify_result(@s.docname(:md) => expected, 
+          @s.docname(:errors) => {})
+      else
+        @ss.search(@c.id).should == true  
+        verify_result(@c.docname(:search) => expected,
+          @c.docname(:search_errors) => {})
+      end
+    end
+    
+    def verify_read_operation_with_error(operation)
+      msg = "Error during #{operation}"
+      @ss.should_receive(:log).with("SourceAdapter raised #{operation} exception: #{msg}")
+      set_test_data('test_db_storage',{},msg,"#{operation} error")
+      if operation == 'query'
+        @ss.read.should == true
+        verify_result(@s.docname(:md) => {},
+          @s.docname(:errors) => {'query-error'=>{'message'=>msg}})
+      else
+        @ss.search(@c.id).should == true
+        verify_result(@c.docname(:search) => {}, 
+          @c.docname(:search_errors) => {'search-error'=>{'message'=>msg}})
+      end
+    end
+  end
+  
+  it "should enqueue process_cud SourceJob" do
+    @s.cud_queue = :cud
+    @ss.process_cud(@c.id)
+    Resque.peek(:cud).should == {"args"=>
+      ["cud", @s.name, @a.name, @u.login, @c.id, nil], "class"=>"Rhosync::SourceJob"}
+  end
+  
+  it "should enqueue process_query SourceJob" do
+    @s.query_queue = :abc
+    @ss.process_query({'foo'=>'bar'})
+    Resque.peek(:abc).should == {"args"=>
+      ["query", @s.name, @a.name, @u.login, nil, {'foo'=>'bar'}], "class"=>"Rhosync::SourceJob"}
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,240 @@
+require 'rubygems'
+require 'rhosync'
+include Rhosync
+
+describe "RhosyncHelper", :shared => true do
+  before(:each) do
+    Store.create
+    Store.db.flushdb
+  end
+end
+
+describe "TestappHelper", :shared => true do
+  before(:all) do
+    @test_app_name = 'application'
+  end
+  def get_testapp_path
+    File.expand_path(File.join(File.dirname(__FILE__),'apps','rhotestapp'))
+  end
+end
+
+describe "RhosyncDataHelper", :shared => true do
+  it_should_behave_like "RhosyncHelper"
+  it_should_behave_like "TestappHelper"
+  
+  before(:each) do
+    @source = 'Product'
+    @user_id = 5
+    @client_id = 1
+    
+    @product1 = {
+      'name' => 'iPhone',
+      'brand' => 'Apple',
+      'price' => '199.99'
+    }
+    
+    @product2 = {
+      'name' => 'G2',
+      'brand' => 'Android',
+      'price' => '99.99'
+    }
+
+    @product3 = {
+      'name' => 'Fuze',
+      'brand' => 'HTC',
+      'price' => '299.99'
+    }
+    
+    @product4 = {
+      'name' => 'Droid',
+      'brand' => 'Android',
+      'price' => '249.99'
+    }
+    
+    @data = {'1'=>@product1,'2'=>@product2,'3'=>@product3}
+  end
+end  
+
+describe "DBObjectsHelper", :shared => true do
+  
+  ERROR = '0_broken_object_id' unless defined? ERROR
+  
+  before(:each) do
+    @a_fields = { :name => @test_app_name }
+    # @a = App.create(@a_fields)
+    @a = (App.load(@test_app_name) || App.create(@a_fields))
+    @u_fields = {:login => 'testuser'}
+    @u = User.create(@u_fields) 
+    @u.password = 'testpass'
+    @c_fields = {
+      :device_type => 'iPhone',
+      :device_pin => 'abcd',
+      :device_port => '3333',
+      :user_id => @u.id,
+      :app_id => @a.id 
+    }
+    @s_fields = {
+      :name => 'SampleAdapter',
+      :url => 'http://example.com',
+      :login => 'testuser',
+      :password => 'testpass',
+    }
+    @s_params = {
+      :user_id => @u.id,
+      :app_id => @a.id
+    }
+    @c = Client.create(@c_fields,{:source_name => @s_fields[:name]})
+    @s = Source.load(@s_fields[:name],@s_params)
+    @s = Source.create(@s_fields,@s_params) if @s.nil?
+    @r = @s.read_state
+    @a.sources << @s.id
+    @a.users << @u.id
+  end
+  
+  def do_post(url,params)
+    post url, params.to_json, {'CONTENT_TYPE'=>'application/json'}
+  end
+  
+  def bulk_data_docname(app_id,user_id)
+    if user_id == "*"
+      File.join(app_id,app_id)
+    else
+      File.join(app_id,user_id,user_id)
+    end
+  end
+  
+  def dump_db_data(store)
+    puts "*"*50 
+    puts "DATA DUMP"
+    puts "*"*50
+    store.db.keys('*').sort.each do |key|
+      next if not key =~ /md|cd/ 
+      line = ""
+      line << "#{key}: "
+      type = store.db.type key
+      if type == 'set'
+        if not key =~ /sources|clients|users/
+          line << "#{store.get_data(key).inspect}"
+        else
+          line << "#{store.db.smembers(key).inspect}"
+        end  
+      else
+        line << "#{store.db.get key}"
+      end
+      puts line
+    end
+    puts "*"*50
+  end
+  
+  def add_client_id(data)
+    res = Marshal.load(Marshal.dump(data))
+    res.each { |key,value| value['rhomobile.rhoclient'] = @c.id.to_s }
+  end
+
+  def add_error_object(data,error_message,error_name='wrongname')
+    error = {'an_attribute'=>error_message,'name'=>error_name} 
+    data.merge!({ERROR=>error})
+    data
+  end
+  
+  def delete_data_directory
+    FileUtils.rm_rf(Rhosync.data_directory)
+  end
+      
+  def set_state(state)
+    state.each do |dockey,data|
+      if data.is_a?(Hash) or data.is_a?(Array)
+        Store.put_data(dockey,data)
+      else
+        Store.put_value(dockey,data)
+      end
+    end
+  end
+  
+  def set_test_data(dockey,data,error_message=nil,error_name='wrongname')
+    if error_message
+      error = {'an_attribute'=>error_message,'name'=>error_name} 
+      data.merge!({ERROR=>error})
+    end  
+    Store.put_data(dockey,data)
+    data
+  end
+  
+  def verify_result(result)
+    result.keys.sort.each do |dockey|
+      expected = result[dockey]
+      begin
+        if expected.is_a?(Hash)
+          Store.get_data(dockey).should == expected
+        elsif expected.is_a?(Array)
+          Store.get_data(dockey,Array).should == expected
+        else
+          Store.get_value(dockey).should == expected
+        end
+      rescue Spec::Expectations::ExpectationNotMetError => e
+        message = "\nVerifying `#{dockey}`\n\n" + e.to_s
+        Kernel::raise(Spec::Expectations::ExpectationNotMetError.new(message))
+      end
+    end
+  end
+  
+  def validate_db(bulk_data,data)
+    validate_db_by_name(bulk_data.dbfile,data)
+  end
+  
+  def validate_db_by_name(name,data)
+    db = SQLite3::Database.new(name)
+    db.execute("select * from sources").each do |row|
+      return false if row.last != get_attrib_counter(data)
+    end
+    db.execute("select * from object_values").each do |row|
+      object = data[row[2]]
+      return false if object.nil? or object[row[1]] != row[3] or row[0] != @s.source_id.to_s
+      object.delete(row[1])
+      data.delete(row[2]) if object.empty?
+    end 
+    data.empty?
+  end
+  
+  def get_attrib_counter(data)
+    counter = {}
+    data.each do |object_name,object|
+      object.each do |attrib,value|
+        counter[attrib] = counter[attrib] ? counter[attrib] + 1 : 1
+      end
+    end
+    BulkDataJob.refs_to_s(counter)
+  end
+  
+  def mock_metadata_method(adapters, &block)
+    adapters.each do |klass|
+      klass.class_eval "def metadata; {'foo'=>'bar'}.to_json; end"
+    end
+    yield
+    adapters.each do |klass|
+      klass.class_eval "def metadata; end"
+    end
+  end
+end
+
+describe "SourceAdapterHelper", :shared => true do
+  it_should_behave_like "RhosyncDataHelper"
+  it_should_behave_like "DBObjectsHelper"
+end
+
+describe "StorageStateHelper", :shared => true do
+  it_should_behave_like "SourceAdapterHelper"
+  
+  before(:each) do
+    @s.name = 'StorageStateAdapter'
+  end
+end
+
+describe "SpecBootstrapHelper", :shared => true do
+  it_should_behave_like "TestappHelper"
+  before(:all) do
+    Rhosync.bootstrap(get_testapp_path) do |rhosync|
+      rhosync.vendor_directory = File.join(File.dirname(__FILE__),'..','vendor')
+    end
+  end
+end

data/spec/store_spec.rb

@@ -0,0 +1,149 @@
+require File.join(File.dirname(__FILE__),'spec_helper')
+
+describe "Store" do
+  
+  it_should_behave_like "SourceAdapterHelper"
+  
+  describe "store methods" do
+    it "should create db class method" do
+      Store.db.class.name.should match(/Redis/)
+    end
+    
+    it "should set redis connection" do
+      begin
+        Store.db = 'localhost:5555'
+        Store.db.server.should == 'localhost:5555'
+      ensure
+        Store.db = ''
+      end
+    end
+    
+    it "should add simple data to new set" do
+      Store.put_data(@s.docname(:md),@data).should == true
+      Store.get_data(@s.docname(:md)).should == @data
+    end
+    
+    it "should add simple array data to new set" do
+      @data = ['1','2','3']
+      Store.put_data(@s.docname(:md),@data).should == true
+      Store.get_data(@s.docname(:md),Array).sort.should == @data
+    end
+  
+    it "should replace simple data to existing set" do
+      new_data,new_data['3'] = {},{'name' => 'Droid','brand' => 'Google'}
+      Store.put_data(@s.docname(:md),@data).should == true
+      Store.put_data(@s.docname(:md),new_data)
+      Store.get_data(@s.docname(:md)).should == new_data
+    end
+    
+    it "should put_value and get_value" do
+      Store.put_value('foo','bar')
+      Store.get_value('foo').should == 'bar'
+    end
+    
+    it "should return true/false if element ismember of a set" do
+      Store.put_data('foo',['a'])
+      Store.ismember?('foo','a').should == true
+      
+      Store.ismember?('foo','b').should == false
+    end
+    
+    it "should return attributes modified in doc2" do
+      Store.put_data(@s.docname(:md),@data).should == true
+      Store.get_data(@s.docname(:md)).should == @data
+    
+      @product3['price'] = '59.99'
+      expected = { '3' => { 'price' => '59.99' } }
+      @data1,@data1['1'],@data1['2'],@data1['3'] = {},@product1,@product2,@product3
+    
+      Store.put_data(@c.docname(:cd),@data1)
+      Store.get_data(@c.docname(:cd)).should == @data1
+      Store.get_diff_data(@s.docname(:md),@c.docname(:cd)).should == [expected,1]
+    end
+  
+    it "should return attributes modified and missed in doc2" do
+      Store.put_data(@s.docname(:md),@data).should == true
+      Store.get_data(@s.docname(:md)).should == @data
+    
+      @product2['price'] = '59.99'
+      expected = { '2' => { 'price' => '99.99' },'3' => @product3 }
+      @data1,@data1['1'],@data1['2'] = {},@product1,@product2
+    
+      Store.put_data(@c.docname(:cd),@data1)
+      Store.get_data(@c.docname(:cd)).should == @data1
+      Store.get_diff_data(@c.docname(:cd),@s.docname(:md)).should == [expected,2]
+    end  
+  
+    it "should ignore reserved attributes" do
+      @newproduct = {
+        'name' => 'iPhone',
+        'brand' => 'Apple',
+        'price' => '199.99',
+        'id' => 1234,
+        'attrib_type' => 'someblob'
+      }
+    
+      @data1 = {'1'=>@newproduct,'2'=>@product2,'3'=>@product3}
+    
+      Store.put_data(@s.docname(:md),@data1).should == true
+      Store.get_data(@s.docname(:md)).should == @data
+    end
+    
+    it "should flash_data" do
+      Store.put_data(@s.docname(:md),@data)
+      Store.flash_data(@s.docname(:md))
+      Store.get_data(@s.docname(:md)).should == {}
+    end
+    
+    it "should get_keys" do
+      expected = ["doc1:1:1:1:source1", "doc1:1:1:1:source2"]
+      Store.put_data(expected[0],@data)
+      Store.put_data(expected[1],@data)
+      Store.get_keys('doc1:1:1:1:*').sort.should == expected
+    end
+    
+    it "should lock document" do
+      doc = "locked_data"
+      m_lock = Store.get_lock(doc)
+      th = Thread.new do 
+        t_lock = Store.get_lock(doc)
+        Store.put_data(doc,{'1'=>@product1},true)
+        Store.release_lock(doc,t_lock) 
+      end
+      Store.put_data(doc,{'2'=>@product2},true)
+      Store.get_data(doc).should == {'2'=>@product2}
+      th.alive?.should == true
+      Store.release_lock(doc,m_lock)
+      sleep(2)
+      m_lock = Store.get_lock(doc)
+      Store.get_data(doc).should == {'1'=>@product1,'2'=>@product2}
+      th.alive?.should == false
+    end
+    
+    it "should lock document in block" do
+      doc = "locked_data"
+      Store.lock(doc,0) do
+        Store.put_data(doc,{'2'=>@product2})
+        Store.get_data(doc).should == {'2'=>@product2}
+      end
+    end
+    
+    it "should create clone of set" do
+      set_state('abc' => @data)
+      Store.clone('abc','def')
+      verify_result('abc' => @data,'def' => @data)
+    end
+    
+    it "should rename a key" do
+      set_state('key1' => @data)
+      Store.rename('key1','key2')
+      verify_result('key1' => {}, 'key2' => @data)
+    end
+    
+    it "should not fail to rename if key doesn't exist" do
+      Store.rename('key1','key2')
+      Store.db.exists('key1').should be_false
+      Store.db.exists('key2').should be_false      
+    end
+  end
+end