rhosync 2.1.16 → 2.1.17.beta1

data/CHANGELOG

@@ -1,3 +1,11 @@
+## 2.1.17 (not yet released)
+* Refactor server middleware. Remove explicit rack dependency. Fix broken rspec examples.
+* Fixing error with recursive loading of application.rb in ruby-1.8.7 and ree
+* #28330213 - Implementing fast_insert/update/delete API
+* #27612327 - Bulk Sync Not Returning Errors from Zendesk ticket #2336
+* #28219647 - Schema Change Error from Zendesk ticket #2353
+* #28328057 - Feature Request: Adding a Bulk Data Job after_perform hook from Zendesk ticket #2367
+
 ## 2.1.16
 * revert rack support for v. 1.4.1 and lock to v. 1.3.6 due to problems with Heroku deployment (Rack 1.4.1 produces 502 error) 
 

data/Rakefile

@@ -66,7 +66,6 @@ begin
     gemspec.files =  FileList["[A-Z]*", "{bench,bin,generators,lib,spec,tasks}/**/*"]
 
     # TODO: Due to https://www.pivotaltracker.com/story/show/3417862, we can't use JSON 1.4.3
-    gemspec.add_dependency "rack", "~> 1.3.6"
     gemspec.add_dependency "sinatra", "~> 1.3.1"
     gemspec.add_dependency "json", "~>1.4.2"
     gemspec.add_dependency "sqlite3-ruby", "~>1.2.5"

data/lib/rhosync.rb

@@ -137,7 +137,7 @@ module Rhosync
     if secret == '<changeme>'                        
       log "*"*60+"\n\n"
       log "WARNING: Change the session secret in config.ru from <changeme> to something secure."
-      log "  i.e. running `rake secret` in a rails app will generate a secret you could use.\n\n"
+      log "  i.e. running `rake rhosync:secret` in your rhosync app directory will generate a secret you could use.\n\n"
       log "*"*60
     end
   end

data/lib/rhosync/api/fast_delete.rb

@@ -0,0 +1,8 @@
+Server.api :fast_delete do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  source_sync = SourceSync.new(source)
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  source_sync.fast_delete(params[:data], timeout,raise_on_expire)
+  'done'
+end

data/lib/rhosync/api/fast_insert.rb

@@ -0,0 +1,8 @@
+Server.api :fast_insert do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  source_sync = SourceSync.new(source)
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  source_sync.fast_insert(params[:data], timeout,raise_on_expire)
+  'done'
+end

data/lib/rhosync/api/fast_update.rb

@@ -0,0 +1,8 @@
+Server.api :fast_update do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  source_sync = SourceSync.new(source)
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  source_sync.fast_update(params[:delete_data], params[:data],timeout,raise_on_expire)
+  'done'
+end

data/lib/rhosync/app.rb

@@ -9,9 +9,6 @@ module Rhosync
     class << self
       def create(fields={})
         fields[:id] = fields[:name]
-        begin
-          require under_score(fields[:name])
-        rescue Exception; end
         super(fields)
       end
     end

data/lib/rhosync/bulk_data/bulk_data.rb

@@ -54,7 +54,11 @@ module Rhosync
       end
       true
     end
-    
+
+    def delete_files
+      FileUtils.rm Dir.glob(File.join(Rhosync.base_directory, "#{self.url}*"))
+    end
+
     class << self
       def create(fields={})
         fields[:id] = fields[:name]

data/lib/rhosync/client_sync.rb

@@ -256,6 +256,21 @@ module Rhosync
         
         if data and data.completed? and data.dbfiles_exist?
           client.update_clientdoc(sources)
+          sources.each do |src|
+            s = Source.load(src, {:user_id => client.user_id, :app_id => client.app_id})
+            errordoc = s.docname(:errors)
+            errors = {}
+            Store.lock(errordoc) do
+              errors = Store.get_data(errordoc)
+            end
+            unless errors.empty?
+              # FIXME: :result => :bulk_sync_error, :errors => "#{errors}"
+              log "Bulk sync errors are found in #{src}: #{errors}"
+              # Delete all related bulk files
+              data.delete_files
+              return {:result => :url, :url => ''}
+            end
+          end
           {:result => :url, :url => data.url}
         elsif data
           {:result => :wait}

data/lib/rhosync/jobs/bulk_data_job.rb

@@ -21,8 +21,8 @@ module Rhosync
           create_hsql_data_file(bulk_data,ts) if Rhosync.blackberry_bulk_sync
           lap_timer('create_hsql_data_file',timer)
           log "finished bulk data process"
-          bulk_data.state = :completed
-          bulk_data.refresh_time = Time.now.to_i + Rhosync.bulk_sync_poll_interval
+          #bulk_data.state = :completed
+          #bulk_data.refresh_time = Time.now.to_i + Rhosync.bulk_sync_poll_interval
         else
           raise Exception.new("No bulk data found for #{params["data_name"]}")
         end
@@ -33,6 +33,26 @@ module Rhosync
         raise e
       end
     end
+
+    def self.after_perform_x(*args)
+      log "BulkDataJob.after_perform_x hook called ..."
+      params = args[0] # 1st parameter is bulk data
+      begin
+        bulk_data = BulkData.load(params["data_name"]) if BulkData.is_exist?(params["data_name"])
+        if bulk_data
+          bulk_data.state = :completed
+          bulk_data.refresh_time = Time.now.to_i + Rhosync.bulk_sync_poll_interval
+          log "BulkDataJob.after_perform_x hook set data state to complete."
+        else 
+          raise Exception.new("No bulk data found for #{params["data_name"]}")
+        end
+      rescue Exception => e
+        bulk_data.delete if bulk_data
+        log "Bulk data after_perform_x raised: #{e.message}"
+        log e.backtrace.join("\n")
+        raise e
+      end
+    end
     
     def self.import_data_to_object_values(db,source)
       data = source.get_data(:md)

data/lib/rhosync/server.rb

@@ -28,13 +28,28 @@ module Rhosync
     set :stats,           false
     
     # default secret
-    @@secret = '<changeme>'
+    @secret = '<changeme>'
                                                                                          
     # Setup route and mimetype for bulk data downloads
     # TODO: Figure out why "mime :data, 'application/octet-stream'" doesn't work
     Rack::Mime::MIME_TYPES['.data'] = 'application/octet-stream'   
      
     include Rhosync
+
+    # Set rhosync middleware
+    set :use_middleware, Proc.new {
+      return false if @middleware_configured # Middleware might be configured only once!
+
+      use Rhosync::BodyContentTypeParser
+      use Rhosync::Stats::Middleware
+      Rhosync::Server.set :secret, @secret unless settings.respond_to?(:secret)
+      use Rack::Session::Cookie,
+          :key => 'rhosync_session',
+          :expire_after => 31536000,
+          :secret => Rhosync::Server.secret
+
+      @middleware_configured ||= true
+    }
                                                               
     helpers do
       def request_action
@@ -145,19 +160,13 @@ module Rhosync
     
     # hook into new so we can enable middleware
     def self.new
-      use Rhosync::BodyContentTypeParser
       if settings.respond_to?(:stats) and settings.send(:stats) == true
-        use Rhosync::Stats::Middleware 
-       	Rhosync.stats = true
+        Rhosync.stats = true
       else
-       	Rhosync::Server.disable :stats
-       	Rhosync.stats = false
+        Rhosync::Server.disable :stats
+        Rhosync.stats = false
       end
-      Rhosync::Server.set :secret, @@secret unless settings.respond_to?(:secret)
-      use Rack::Session::Cookie, 
-          :key => 'rhosync_session',
-          :expire_after => 31536000,
-          :secret => Rhosync::Server.secret
+      settings.use_middleware      
       super
     end
         

data/lib/rhosync/source_sync.rb

@@ -81,6 +81,29 @@ module Rhosync
         @source.app_id,@source.user_id,client_id,params)
     end
     
+    def fast_insert(new_objs, timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        diff_count = new_objs.size
+        @source.put_data(:md, new_objs, true)
+        @source.update_count(:md_size,diff_count)
+      end
+    end
+    
+    def fast_update(orig_hash, new_hash, timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        @source.delete_data(:md, orig_hash)
+        @source.put_data(:md, new_hash, true)
+      end
+    end 
+    
+    def fast_delete(delete_objs, timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        diff_count = -delete_objs.size
+        @source.delete_data(:md, delete_objs)
+        @source.update_count(:md_size,diff_count)
+      end
+    end
+    
     def push_objects(objects,timeout=10,raise_on_expire=false,rebuild_md=true)
       @source.lock(:md,timeout,raise_on_expire) do |s|
         diff_count = 0
@@ -259,7 +282,14 @@ module Rhosync
         data = @adapter.send(method) 
         if data
           @source.put_value(method,data)
-          @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
+          if method == :schema
+            parsed = JSON.parse(data)
+            schema_version = parsed['version']
+            raise "Mandatory version key is not defined in source adapter schema method" if schema_version.nil? 
+            @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(schema_version))
+          else
+            @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
+          end
         end
       end
     end

data/lib/rhosync/stats/middleware.rb

@@ -6,14 +6,19 @@ module Rhosync
       end
 
       def call(env)
-        start = Time.now.to_f
-        status, headers, body = @app.call(env)
-        finish = Time.now.to_f
-        metric = "http:#{env['REQUEST_METHOD']}:#{env['PATH_INFO']}"
-        source_name = env['rack.request.query_hash']["source_name"] if env['rack.request.query_hash']
-        metric << ":#{source_name}" if source_name
-        Record.save_average(metric,finish - start)
-        [status, headers, body]
+        if Rhosync.stats || Rhosync::Server.stats
+          start = Time.now.to_f
+          status, headers, body = @app.call(env)
+          finish = Time.now.to_f
+          metric = "http:#{env['REQUEST_METHOD']}:#{env['PATH_INFO']}"
+          source_name = env['rack.request.query_hash']["source_name"] if env['rack.request.query_hash']
+          metric << ":#{source_name}" if source_name
+          Record.save_average(metric,finish - start)
+          [status, headers, body]
+        else
+          status, headers, body = @app.call(env)
+          [status, headers, body]
+        end
       end
     end
   end

data/lib/rhosync/version.rb

@@ -1,3 +1,3 @@
 module Rhosync
-  VERSION = '2.1.16'
+  VERSION = '2.1.17.beta1'
 end

data/spec/api/api_helper.rb

@@ -24,6 +24,8 @@ describe "ApiHelper", :shared => true do
   end
 
   def app
+    Rhosync::Server.set :stats, false
+    Rhosync.stats = false
     @app ||= Rhosync::Server.new
   end
 

data/spec/api/fast_delete_spec.rb

@@ -0,0 +1,27 @@
+require File.join(File.dirname(__FILE__),'api_helper')
+
+describe "RhoconnectApiFastDelete" do
+  it_should_behave_like "ApiHelper"
+  
+  it "should delete an object from rhosync's :md" do
+    data = {'1' => @product1, '2' => @product2, '3' => @product3}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '3')
+    post "/api/fast_delete", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :data => {'3' => @product3}
+    last_response.should be_ok
+    data.delete('3')
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'2')
+  end
+
+  it "should not properly delete the object if fast_delete is called without all the attributes (because fast_delete doesn't ensure any data integrity)" do
+    data = {'1' => @product1, '2' => @product2, '3' => @product3}
+    delete_data = {'3' => {'price' => '1.99'}}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '3')
+    post "/api/fast_delete", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :data => delete_data
+    last_response.should be_ok
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'2')
+  end
+end

data/spec/api/fast_insert_spec.rb

@@ -0,0 +1,28 @@
+require File.join(File.dirname(__FILE__),'api_helper')
+
+describe "RhoconnectApiFastInsert" do
+  it_should_behave_like "ApiHelper"
+  
+  it "should append new objects to rhosync's :md" do
+    data = {'1' => @product1, '2' => @product2}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '2')
+    post "/api/fast_insert", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :data => {'3' => @product3}
+    last_response.should be_ok
+    data.merge!({'3' => @product3})
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'3')
+  end
+
+  it "should incorrectly append data to existing object (because fast_insert doesn't ensure any data integrity)" do
+    data = {'1' => @product1, '2' => @product2, '3' => @product3}
+    incorrect_insert = {'3' => {'price' => '1.99', 'new_field' => 'value'}}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '3')
+    post "/api/fast_insert", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :data => incorrect_insert
+    last_response.should be_ok
+    data['3'].merge!(incorrect_insert['3'])
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'4')
+  end
+end

data/spec/api/fast_update_spec.rb

@@ -0,0 +1,51 @@
+require File.join(File.dirname(__FILE__),'api_helper')
+
+describe "RhoconnectApiFastUpdate" do
+  it_should_behave_like "ApiHelper"
+  
+  it "should update an attribute and add new one for an object in rhosync's :md" do
+    data = {'1' => @product1, '2' => @product2, '3' => @product3}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '3')
+    
+    orig_obj_attrs = {'3' => {'price' => @product3['price']}}
+    new_obj_attrs = {'3' => {'price' => '0.99', 'new_attr' => 'new_value'}}
+    
+    post "/api/fast_update", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :delete_data => orig_obj_attrs, :data => new_obj_attrs
+    last_response.should be_ok
+    data['3'].merge!(new_obj_attrs['3'])
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'3')
+  end
+  
+  it "should update one attr, add one attr, and remove one attr for an object in rhosync's :md" do
+    data = {'1' => @product1, '2' => @product2, '3' => @product3}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '3')
+    
+    orig_obj_attrs = {'3' => {'name' => @product3['name'], 'price' => @product3['price']}}
+    new_obj_attrs = {'3' => {'price' => '0.99', 'new_attr' => 'new_value'}}
+    
+    post "/api/fast_update", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :delete_data => orig_obj_attrs, :data => new_obj_attrs
+    last_response.should be_ok
+    data['3'].delete('name')
+    data['3'].merge!(new_obj_attrs['3'])
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'3')
+  end
+  
+  it "should remove all attributes , but leave the count incorrect (because fast_update doesn't check if the whole object is deleted)" do
+    data = {'1' => @product1, '2' => @product2, '3' => @product3}
+    @s = Source.load(@s_fields[:name],@s_params)
+    set_state(@s.docname(:md) => data,@s.docname(:md_size) => '3')
+    
+    orig_obj_attrs = {'3' => @product3}
+    new_obj_attrs = {}
+    
+    post "/api/fast_update", :api_token => @api_token, 
+      :user_id => @u.id, :source_id => @s_fields[:name], :delete_data => orig_obj_attrs, :data => new_obj_attrs
+    last_response.should be_ok
+    data.delete('3')
+    verify_result(@s.docname(:md) => data,@s.docname(:md_size)=>'3')
+  end
+end

data/spec/api/stats_spec.rb

@@ -9,10 +9,12 @@ describe "RhosyncApiStats" do
   
   before(:each) do
     Rhosync::Server.set :stats, true
+    Rhosync.stats = true
   end
   
   after(:each) do
     Rhosync::Server.set :stats, false
+    Rhosync.stats = false
   end
   
   it "should retrieve metric names" do

data/spec/client_sync_spec.rb

@@ -9,6 +9,9 @@ describe "ClientSync" do
     lambda { ClientSync.new(nil,@c,2) }.should raise_error(ArgumentError,'Unknown source')
   end
   
+  let(:mock_schema) { {"property" => { "name" => "string", "brand" => "string" }, "version" => "1.0"} }
+  let(:sha1) { get_sha1(mock_schema['version']) }
+
   before(:each) do
     @s = Source.load(@s_fields[:name],@s_params)
     @cs = ClientSync.new(@s,@c,2)
@@ -372,7 +375,6 @@ describe "ClientSync" do
       token1.should be_nil
       Store.get_data(@c.docname(:search)).should == {}
     end
-     
   end
   
   describe "page methods" do
@@ -380,10 +382,20 @@ describe "ClientSync" do
       Store.put_data(@s.docname(:md),@data).should == true
       Store.get_data(@s.docname(:md)).should == @data
       Store.put_value(@s.docname(:md_size),@data.size)
-      @expected = {'1'=>@product1,'2'=>@product2}
-      @cs.compute_page.should == [0,3,@expected]
+
+      progress_count, total_count, res = @cs.compute_page
+      progress_count.to_i.should == 0
+      total_count.to_i.should == 3
+      res.each do |key, value|
+        @data.has_key?(key).should == true
+        @data[key].should == value
+      end
+      
       Store.get_value(@cs.client.docname(:cd_size)).to_i.should == 0
-      Store.get_data(@cs.client.docname(:page)).should == @expected      
+      Store.get_data(@cs.client.docname(:page)).each do |key, value|
+        @data.has_key?(key).should == true
+        @data[key].should == value
+      end
     end
   
     it "appends diff to the client document" do
@@ -483,7 +495,7 @@ describe "ClientSync" do
         token = @c.get_value(:page_token)
         result.should ==  [{"version"=>ClientSync::VERSION},{"token"=>token}, 
           {"count"=>1}, {"progress_count"=>0},{"total_count"=>1},{'insert'=>expected}]
-        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_sha1).should == sha1
       end
     end
     
@@ -494,8 +506,8 @@ describe "ClientSync" do
         token = @c.get_value(:page_token)
         result.should ==  [{"version"=>ClientSync::VERSION},{"token"=>token}, 
           {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{'schema-changed'=>'true'}]
-        @c.get_value(:schema_page).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
-        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_page).should == sha1
+        @c.get_value(:schema_sha1).should == sha1
       end
     end
     
@@ -506,8 +518,8 @@ describe "ClientSync" do
         token = @c.get_value(:page_token)
         @cs.send_cud.should ==  [{"version"=>ClientSync::VERSION},{"token"=>token}, 
           {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{'schema-changed'=>'true'}]
-        @c.get_value(:schema_page).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
-        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_page).should == sha1
+        @c.get_value(:schema_sha1).should == sha1        
       end
     end
     
@@ -519,7 +531,7 @@ describe "ClientSync" do
         @cs.send_cud(token).should ==  [{"version"=>ClientSync::VERSION},{"token"=>""}, 
           {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{}]
         @c.get_value(:schema_page).should be_nil
-        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_sha1).should == sha1
       end
     end
     
@@ -538,7 +550,7 @@ describe "ClientSync" do
         @cs.send_cud(token).should ==  [{"version"=>ClientSync::VERSION},{"token"=>""}, 
           {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{}]
         @c.get_value(:schema_page).should be_nil
-        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_sha1).should == sha1
         data = BulkData.load(docname)
         data.refresh_time.should <= Time.now.to_i
       end 
@@ -574,6 +586,7 @@ describe "ClientSync" do
       set_state('test_db_storage' => @data)
       ClientSync.bulk_data(:user,@c)
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,@u.id))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,@u.id))
       ClientSync.bulk_data(:user,@c).should == {:result => :url,
         :url => BulkData.load(bulk_data_docname(@a.id,@u.id)).url}
       verify_result(
@@ -581,6 +594,18 @@ describe "ClientSync" do
         "source:#{@a_fields[:name]}:#{@u_fields[:login]}:#{@s_fields[:name]}:md" => @data,
         "source:#{@a_fields[:name]}:#{@u_fields[:login]}:#{@s_fields[:name]}:md_copy" => @data)
     end
+
+    it "should return empty bulk data url if there are errors in query" do
+      ClientSync.bulk_data(:user,@c)
+      BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,@u.id))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,@u.id))
+      errordoc = @s.docname(:errors) # source SampleAdapter
+      operation = 'query'
+      Store.lock(errordoc) do
+        Store.put_data(errordoc,{"#{operation}-error"=>{'message'=>"Some exception message"}}, true)
+      end
+      ClientSync.bulk_data(:user,@c).should == {:result => :url, :url => ''}
+    end
     
     it "should escape bulk data url" do
       name = 'a b'
@@ -590,6 +615,7 @@ describe "ClientSync" do
         :user_id => name,
         :sources => [@s_fields[:name]])
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,name))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,name))
       data = BulkData.load(bulk_data_docname(@a.id,name))
       data.url.should match /a%20b/
       data.delete
@@ -600,6 +626,7 @@ describe "ClientSync" do
       @s.partition = :app
       ClientSync.bulk_data(:app,@c)
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,"*"))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,"*"))
       ClientSync.bulk_data(:app,@c).should == {:result => :url,
         :url => BulkData.load(bulk_data_docname(@a.id,"*")).url}
       verify_result(
@@ -613,6 +640,7 @@ describe "ClientSync" do
       @s.sync_type = :bulk_sync_only
       ClientSync.bulk_data(:user,@c)
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,@u.id))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,@u.id))
       ClientSync.bulk_data(:user,@c).should == {:result => :url,
         :url => BulkData.load(bulk_data_docname(@a.id,@u.id)).url}
       verify_result(
@@ -626,6 +654,7 @@ describe "ClientSync" do
       Rhosync.blackberry_bulk_sync = true
       ClientSync.bulk_data(:user,@c)
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,@u.id))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,@u.id))
       data = BulkData.load(bulk_data_docname(@a.id,@u.id))
       ClientSync.bulk_data(:user,@c).should == {:result => :url, :url => data.url}
       File.delete(data.dbfile)

data/spec/jobs/bulk_data_job_spec.rb

@@ -12,6 +12,8 @@ describe "BulkDataJob" do
     delete_data_directory
   end
   
+  let(:mock_schema) { {"property" => { "name" => "string", "brand" => "string" }, "version" => "1.0"} }
+  
   it "should create bulk data files from master document" do
     set_state('test_db_storage' => @data)
     docname = bulk_data_docname(@a.id,@u.id)
@@ -24,6 +26,7 @@ describe "BulkDataJob" do
       :user_id => @u.id,
       :sources => [@s_fields[:name], 'FixedSchemaAdapter'])
     BulkDataJob.perform("data_name" => data.name)
+    BulkDataJob.after_perform_x("data_name" => data.name)
     data = BulkData.load(docname)
     data.completed?.should == true
     verify_result(@s.docname(:md) => @data,@s.docname(:md_copy) => @data)
@@ -51,6 +54,7 @@ describe "BulkDataJob" do
       :user_id => @u.id,
       :sources => [@s_fields[:name]])
     BulkDataJob.perform("data_name" => data.name)
+    BulkDataJob.after_perform_x("data_name" => data.name)
     data = BulkData.load(docname)
     data.completed?.should == true
     verify_result(@s.docname(:md) => @data,@s.docname(:md_copy) => @data)
@@ -69,6 +73,7 @@ describe "BulkDataJob" do
         :user_id => @u.id,
         :sources => [@s_fields[:name]])
       BulkDataJob.perform("data_name" => data.name)
+      BulkDataJob.after_perform_x("data_name" => data.name)
       data = BulkData.load(docname)
       data.completed?.should == true
       verify_result(@s.docname(:md) => @data,
@@ -88,11 +93,11 @@ describe "BulkDataJob" do
         :user_id => @u.id,
         :sources => [@s_fields[:name]])
       BulkDataJob.perform("data_name" => data.name)
+      BulkDataJob.after_perform_x("data_name" => data.name)
       data = BulkData.load(docname)
       data.completed?.should == true
-      verify_result(@s.docname(:md) => @data,
-        @s.docname(:schema) => "{\"property\":{\"brand\":\"string\",\"name\":\"string\"},\"version\":\"1.0\"}",
-        @s.docname(:md_copy) => @data)
+      verify_result(@s.docname(:md) => @data, @s.docname(:md_copy) => @data)
+      JSON.parse(Store.get_value(@s.docname(:schema))).should == mock_schema      
       validate_db(data,@s.name => @data).should == true
     end
   end
@@ -108,7 +113,9 @@ describe "BulkDataJob" do
   end
   
   it "should delete bulk data if exception is raised" do
-    lambda { BulkDataJob.perform("data_name" => 'broken') }.should raise_error(Exception)
+    lambda { 
+      BulkDataJob.perform("data_name" => 'broken')
+      BulkDataJob.after_perform_x("data_name" => data.name) }.should raise_error(Exception)
     Store.db.keys('bulk_data*').should == []
   end
   
@@ -118,7 +125,9 @@ describe "BulkDataJob" do
       :app_id => 'broken',
       :user_id => @u.id,
       :sources => [@s_fields[:name]])
-    lambda { BulkDataJob.perform("data_name" => data.name) }.should raise_error(Exception)
+    lambda { 
+      BulkDataJob.perform("data_name" => data.name)
+      BulkDataJob.after_perform_x("data_name" => data.name) }.should raise_error(Exception)
     Store.db.keys('bulk_data*').should == []
   end
 end

data/spec/perf/bulk_data_perf_spec.rb

@@ -27,6 +27,7 @@ describe "BulkData Performance" do
       :user_id => @u.id,
       :sources => [@s_fields[:name]])
     BulkDataJob.perform("data_name" => data.name)
+    BulkDataJob.after_perform_x("data_name" => data.name)
     lap_timer('BulkDataJob.perform duration',start)
   end
 end

data/spec/ping/apple_spec.rb

@@ -41,18 +41,24 @@ describe "Ping Apple" do
   end
   
   it "should compute apn_message" do
-    expected = <<-eos
-\000\000 \253\315\000g{"aps":{"vibrate":"5","badge":5,"sound":"hello.mp3","alert":"hello world"},"do_sync":["SampleAdapter"]}
-eos
-    Apple.apn_message(@params).should == expected.strip!
+    expected_hash = {
+      "aps"=>{"vibrate"=>"5", "badge"=>5, "alert"=>"hello world", "sound"=>"hello.mp3"},
+      "do_sync"=>["SampleAdapter"]
+    }
+    apn_message = Apple.apn_message(@params)
+    apn_message.start_with?("\000\000 \253\315\000g").should be_true
+    JSON.parse(apn_message.sub("\000\000 \253\315\000g","")).should ==  expected_hash
   end
   
   it "should compute apn_message with source array" do
     @params['sources'] << 'SimpleAdapter'
-    expected = <<-eos
-\000\000 \253\315\000w{"aps":{"vibrate":"5","badge":5,"sound":"hello.mp3","alert":"hello world"},"do_sync":["SampleAdapter","SimpleAdapter"]}
-eos
-    Apple.apn_message(@params).should == expected.strip!
+    expected_hash = {
+      "aps"=>{"vibrate"=>"5", "badge"=>5, "alert"=>"hello world", "sound"=>"hello.mp3"},
+      "do_sync"=>["SampleAdapter", "SimpleAdapter"]
+    }
+    apn_message = Apple.apn_message(@params)
+    apn_message.start_with?("\000\000 \253\315\000w").should be_true
+    JSON.parse(apn_message.sub("\000\000 \253\315\000w","")).should ==  expected_hash
   end
   
   it "should raise SocketError if socket fails" do

data/spec/server/server_spec.rb

@@ -70,14 +70,6 @@ describe "Server" do
     Rhosync::Server.secret.should == "secure!"
   end
   
-  it "should use Stats::Middleware if stats enabled" do
-    Rhosync::Server.enable :stats
-    Rhosync::Server.new
-    Rhosync.stats.should == true
-    Rhosync.stats = nil
-    Rhosync::Server.disable :stats
-  end
-  
   it "should update session secret to default" do
     Rhosync::Server.set :secret, "<changeme>"
     Rhosync::Server.secret.should == "<changeme>"
@@ -380,6 +372,7 @@ describe "Server" do
       set_state('test_db_storage' => @data)
       get "/application/bulk_data", :partition => :user, :client_id => @c.id
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,@u.id))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,@u.id))
       get "/application/bulk_data", :partition => :user, :client_id => @c.id
       last_response.should be_ok
       data = BulkData.load(bulk_data_docname(@a.id,@u.id))
@@ -392,6 +385,7 @@ describe "Server" do
       set_state('test_db_storage' => @data)
       get "/application/bulk_data", :partition => :user, :client_id => @c.id
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,@u.id))
+      BulkDataJob.after_perform_x("data_name" => bulk_data_docname(@a.id,@u.id))
       get "/application/bulk_data", :partition => :user, :client_id => @c.id
       get JSON.parse(last_response.body)["url"]
       last_response.should be_ok

data/spec/source_sync_spec.rb

@@ -9,6 +9,8 @@ describe "SourceSync" do
     @ss = SourceSync.new(@s)
   end
   
+  let(:mock_schema) { {"property" => { "name" => "string", "brand" => "string" }, "version" => "1.0"} }
+
   it "should create SourceSync" do
     @ss.adapter.is_a?(SampleAdapter).should == true
   end
@@ -78,12 +80,25 @@ describe "SourceSync" do
         expected = {'1'=>@product1,'2'=>@product2}
         set_state('test_db_storage' => expected)
         @ss.process_query
-        verify_result(@s.docname(:md) => expected,
-          @s.docname(:schema) => "{\"property\":{\"brand\":\"string\",\"name\":\"string\"},\"version\":\"1.0\"}",
-          @s.docname(:schema_sha1) => "8c148c8c1a66c7baf685c07d58bea360da87981b")
+        verify_result(@s.docname(:md) => expected)
+        JSON.parse(Store.get_value(@s.docname(:schema))).should == mock_schema
+        Store.get_value(@s.docname(:schema_sha1)).should == get_sha1(mock_schema['version'])
       end
     end
-    
+
+    it "should raise exception if source adapter schema has no version key/value pair" do
+      mock_schema_no_version_method([SampleAdapter]) do
+        expected = {'1'=>@product1,'2'=>@product2}
+        set_state('test_db_storage' => expected)
+        @ss.process_query
+         errordoc = @s.docname(:errors)
+         errors = {}
+         Store.lock(errordoc) { errors = Store.get_data(errordoc) }
+         errors.empty?().should == false
+         errors["query-error"]["message"].should == "Mandatory version key is not defined in source adapter schema method"
+      end
+    end
+
     it "should process source adapter with stash" do
       expected = {'1'=>@product1,'2'=>@product2}
       set_state('test_db_storage' => expected)
@@ -152,7 +167,6 @@ describe "SourceSync" do
         verify_result(
           @c.docname(:update_errors) =>
             {"#{ERROR}-error"=>{"message"=>msg}, ERROR=>data[ERROR]},
-          @c.docname(:update) => {'4'=> { 'price' => '199.99'}},
           @c.docname(:update_rollback) => {ERROR=>{"price"=>"99.99"}})
       end
     end
@@ -174,9 +188,7 @@ describe "SourceSync" do
         data = add_error_object({'2'=>@product2},msg)
         set_state(@c.docname(:delete) => data)
         @ss.delete(@c.id)
-        verify_result(@c.docname(:delete_errors) => 
-          {"#{ERROR}-error"=>{"message"=>msg}, ERROR=>data[ERROR]},
-            @c.docname(:delete) => {'2'=>@product2})
+        verify_result(@c.docname(:delete_errors) => {"#{ERROR}-error"=>{"message"=>msg}, ERROR=>data[ERROR]})
       end
     end
     

data/spec/spec_helper.rb

@@ -194,6 +194,23 @@ module TestHelpers
     end
   end
   
+  def mock_schema_no_version_method(adapters, &block)
+    adapters.each do |klass|
+      klass.class_eval 'def schema
+        {
+          "property" => {
+            "name" => "string",
+            "brand" => "string"
+          }
+        }.to_json
+      end'
+    end
+    yield
+    adapters.each do |klass|
+      klass.class_eval "def schema; end"
+    end
+  end
+
   def unzip_file(file,file_dir)
     Zip::ZipFile.open(file) do |zip_file|
       zip_file.each do |f|
@@ -203,6 +220,11 @@ module TestHelpers
       end
     end
   end
+
+  def get_sha1(str)
+    Digest::SHA1.hexdigest(str)
+  end  
+  
 end #TestHelpers
 
 describe "RhosyncHelper", :shared => true do

data/spec/stats/middleware_spec.rb

@@ -1,4 +1,7 @@
+require File.join(File.dirname(__FILE__),'..','spec_helper')
+require File.join(File.dirname(__FILE__),'..','..','lib','rhosync','server.rb')
 require 'rhosync'
+
 STATS_RECORD_RESOLUTION = 2 unless defined? STATS_RECORD_RESOLUTION
 STATS_RECORD_SIZE = 8 unless defined? STATS_RECORD_SIZE
 
@@ -12,10 +15,17 @@ describe "Middleware" do
     Store.db.flushdb
     app = mock('app')
     app.stub!(:call)
+    Rhosync.stats = true
+    Rhosync::Server.enable :stats
     @middleware = Middleware.new(app)
     Store.stub!(:lock).and_yield
   end
-  
+
+  after(:each) do
+    Rhosync.stats = false
+    Rhosync::Server.disable :stats
+  end
+
   it "should compute http average" do
     Time.stub!(:now).and_return { @now += 0.3; @now }
     env = {

metadata

@@ -1,13 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: rhosync
 version: !ruby/object:Gem::Version 
-  hash: 43
-  prerelease: 
+  hash: 391570199
+  prerelease: 7
   segments: 
   - 2
   - 1
-  - 16
-  version: 2.1.16
+  - 17
+  - beta
+  - 1
+  version: 2.1.17.beta1
 platform: ruby
 authors: 
 - Rhomobile
@@ -15,28 +17,12 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-07 00:00:00 Z
+date: 2012-04-24 00:00:00 Z
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: rack
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 3
-        - 6
-        version: 1.3.6
-  type: :runtime
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: sinatra
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -48,11 +34,11 @@ dependencies:
         - 1
         version: 1.3.1
   type: :runtime
-  version_requirements: *id002
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: json
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -64,11 +50,11 @@ dependencies:
         - 2
         version: 1.4.2
   type: :runtime
-  version_requirements: *id003
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: sqlite3-ruby
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -80,11 +66,11 @@ dependencies:
         - 5
         version: 1.2.5
   type: :runtime
-  version_requirements: *id004
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: rubyzip
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -96,11 +82,11 @@ dependencies:
         - 4
         version: 0.9.4
   type: :runtime
-  version_requirements: *id005
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: uuidtools
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -112,11 +98,11 @@ dependencies:
         - 1
         version: 2.1.1
   type: :runtime
-  version_requirements: *id006
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: redis
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -128,11 +114,11 @@ dependencies:
         - 1
         version: 2.1.1
   type: :runtime
-  version_requirements: *id007
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: resque
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -144,11 +130,11 @@ dependencies:
         - 0
         version: 1.14.0
   type: :runtime
-  version_requirements: *id008
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: rest-client
   prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -160,11 +146,11 @@ dependencies:
         - 1
         version: 1.6.1
   type: :runtime
-  version_requirements: *id009
+  version_requirements: *id008
 - !ruby/object:Gem::Dependency 
   name: templater
   prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
+  requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -176,11 +162,11 @@ dependencies:
         - 0
         version: 1.0.0
   type: :runtime
-  version_requirements: *id010
+  version_requirements: *id009
 - !ruby/object:Gem::Dependency 
   name: rake
   prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
+  requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -193,11 +179,11 @@ dependencies:
         - 2
         version: 0.9.2.2
   type: :runtime
-  version_requirements: *id011
+  version_requirements: *id010
 - !ruby/object:Gem::Dependency 
   name: log4r
   prerelease: false
-  requirement: &id012 !ruby/object:Gem::Requirement 
+  requirement: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -209,11 +195,11 @@ dependencies:
         - 7
         version: 1.1.7
   type: :development
-  version_requirements: *id012
+  version_requirements: *id011
 - !ruby/object:Gem::Dependency 
   name: jeweler
   prerelease: false
-  requirement: &id013 !ruby/object:Gem::Requirement 
+  requirement: &id012 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -225,11 +211,11 @@ dependencies:
         - 0
         version: 1.4.0
   type: :development
-  version_requirements: *id013
+  version_requirements: *id012
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
-  requirement: &id014 !ruby/object:Gem::Requirement 
+  requirement: &id013 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -241,11 +227,11 @@ dependencies:
         - 0
         version: 1.3.0
   type: :development
-  version_requirements: *id014
+  version_requirements: *id013
 - !ruby/object:Gem::Dependency 
   name: rcov
   prerelease: false
-  requirement: &id015 !ruby/object:Gem::Requirement 
+  requirement: &id014 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -257,11 +243,11 @@ dependencies:
         - 8
         version: 0.9.8
   type: :development
-  version_requirements: *id015
+  version_requirements: *id014
 - !ruby/object:Gem::Dependency 
   name: faker
   prerelease: false
-  requirement: &id016 !ruby/object:Gem::Requirement 
+  requirement: &id015 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -273,11 +259,11 @@ dependencies:
         - 1
         version: 0.3.1
   type: :development
-  version_requirements: *id016
+  version_requirements: *id015
 - !ruby/object:Gem::Dependency 
   name: rack-test
   prerelease: false
-  requirement: &id017 !ruby/object:Gem::Requirement 
+  requirement: &id016 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -289,11 +275,11 @@ dependencies:
         - 3
         version: 0.5.3
   type: :development
-  version_requirements: *id017
+  version_requirements: *id016
 - !ruby/object:Gem::Dependency 
   name: thor
   prerelease: false
-  requirement: &id018 !ruby/object:Gem::Requirement 
+  requirement: &id017 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -305,7 +291,7 @@ dependencies:
         - 6
         version: 0.13.6
   type: :development
-  version_requirements: *id018
+  version_requirements: *id017
 description: RhoSync Synchronization Framework and related command-line utilities
 email: dev@rhomobile.com
 executables: 
@@ -368,6 +354,9 @@ files:
 - lib/rhosync/api/create_user.rb
 - lib/rhosync/api/delete_client.rb
 - lib/rhosync/api/delete_user.rb
+- lib/rhosync/api/fast_delete.rb
+- lib/rhosync/api/fast_insert.rb
+- lib/rhosync/api/fast_update.rb
 - lib/rhosync/api/get_api_token.rb
 - lib/rhosync/api/get_client_params.rb
 - lib/rhosync/api/get_db_doc.rb
@@ -484,6 +473,9 @@ files:
 - spec/api/create_user_spec.rb
 - spec/api/delete_client_spec.rb
 - spec/api/delete_user_spec.rb
+- spec/api/fast_delete_spec.rb
+- spec/api/fast_insert_spec.rb
+- spec/api/fast_update_spec.rb
 - spec/api/get_api_token_spec.rb
 - spec/api/get_client_params_spec.rb
 - spec/api/get_db_doc_spec.rb
@@ -579,16 +571,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: 25
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.20
 signing_key: 
 specification_version: 3
 summary: RhoSync Synchronization Framework