rhosync 2.0.0.beta3 → 2.0.0.beta4

data/lib/rhosync/jobs/bulk_data_job.rb

@@ -48,6 +48,50 @@ module Rhosync
       end
       counter
     end
+    
+    # Loads data into fixed schema table based on source settings
+    def self.import_data_to_fixed_schema(db,source)
+      data = source.get_data(:md)
+      counter = {}
+      columns,qm = [],[]
+      create_table = ['object varchar']
+      schema = JSON.parse(source.schema)
+      
+      db.transaction do |database|
+        # Create a table with columns specified by 'property' array in settings
+        schema['property'].each do |column|
+          create_table << "#{column.keys[0]} varchar default NULL" 
+          columns << column.keys[0]
+          qm << '?'
+        end
+        database.execute("CREATE TABLE #{source.name}(
+          #{create_table.join(",")} );")
+        
+        # Insert each object as single row in fixed schema table
+        database.prepare("insert into #{source.name} 
+          (object,#{columns.join(',')}) values (?,#{qm.join(',')})") do |stmt|
+          data.each do |obj,row|
+            args = [obj]
+            columns.each do |col|
+              args << row[col]
+            end  
+            stmt.execute(args)
+          end
+        end
+        
+        # Create indexes for specified columns in settings 'index'
+        schema['index'].each do |index|
+          database.execute("CREATE INDEX #{index.keys[0]} on #{source.name} (#{index.values[0]});")
+        end if schema['index']
+        
+        # Create unique indexes for specified columns in settings 'unique_index'
+        schema['unique_index'].each do |index|
+          database.execute("CREATE UNIQUE INDEX #{index.keys[0]} on #{source.name} (#{index.values[0]});")
+        end if schema['unique_index']
+      end
+    
+      return {}
+    end
      
     def self.refs_to_s(refs)
       str = ''
@@ -84,7 +128,12 @@ module Rhosync
           :user_id => bulk_data.user_id})
         source.source_id = src_counter
         src_counter += 1
-        source_attrib_refs = import_data_to_object_values(db,source)
+        source_attrib_refs = nil
+        if source.schema
+          source_attrib_refs = import_data_to_fixed_schema(db,source)
+        else
+          source_attrib_refs = import_data_to_object_values(db,source)
+        end
         sources_refs[source_name] = 
           {:source => source, :refs => source_attrib_refs}
         lap_timer("finished importing sqlite data for #{source_name}",timer)
@@ -99,7 +148,7 @@ module Rhosync
       hsql_file = dbfile + ".hsqldb"
       raise Exception.new("Error running hsqldata") unless 
         system('java','-cp', File.join(File.dirname(__FILE__),'..','..','..','vendor','hsqldata.jar'),
-        'com.rhomobile.hsqldata.HsqlData', dbfile, hsql_file, schema, index)
+        'com.rhomobile.hsqldata.HsqlData', dbfile, hsql_file)
     end
     
     def self.get_file_args(bulk_data_name,ts)

data/lib/rhosync/server.rb

@@ -112,10 +112,6 @@ module Rhosync
             params[:source_name] ? {:source_name => current_source.name} : {:source_name => '*'}) 
         end  
       end
-      
-      def source_config
-        { "sources" => Rhosync.get_config(Rhosync.base_directory)[:sources] }
-      end
 
       def catch_all
         begin
@@ -136,15 +132,21 @@ module Rhosync
     Rhosync.log "Rhosync Server v#{Rhosync::VERSION} started..."
 
     before do
-      if params["cud"]
-        cud = JSON.parse(params["cud"])
-        params.delete("cud")
-        params.merge!(cud)
-      end
-      if request.env['CONTENT_TYPE'] == 'application/json'
-        params.merge!(JSON.parse(request.body.read))
-        request.body.rewind
-      end      
+      begin
+        if params["cud"]
+          cud = JSON.parse(params["cud"])
+          params.delete("cud")
+          params.merge!(cud)
+        end
+        if request.env['CONTENT_TYPE'] == 'application/json'
+          params.merge!(JSON.parse(request.body.read))
+          request.body.rewind
+        end      
+      rescue JSON::ParserError => jpe
+        throw :halt, [500, "Server error while processing client data"]
+      rescue Exception => e
+        throw :halt, [500, "Internal server error"]
+      end
       if params[:version] and params[:version].to_i < 3
         throw :halt, [404, "Server supports version 3 or higher of the protocol."]
       end

data/lib/rhosync/source.rb

@@ -13,6 +13,7 @@ module Rhosync
     field :queue,:string
     field :query_queue,:string
     field :cud_queue,:string
+    field :schema, :string
     attr_accessor :app_id, :user_id
     validates_presence_of :name #, :source_id
     
@@ -27,6 +28,7 @@ module Rhosync
       fields[:partition_type] ||= :user
       fields[:poll_interval] ||= 300
       fields[:sync_type] ||= :incremental
+      fields[:schema] = fields[:schema].to_json if fields[:schema]
     end
         
     def self.create(fields,params)
@@ -43,6 +45,7 @@ module Rhosync
     end
     
     def update(fields)
+      fields = fields.with_indifferent_access # so we can access hash keys as symbols
       self.class.set_defaults(fields)
       super(fields)
     end

data/lib/rhosync/test_methods.rb

@@ -4,6 +4,8 @@ module Rhosync
   end
   
   module TestMethods
+    # Initializes the source adapter under test for a given user, typically in a before(:each) block
+    # setup_test_for(Product,'testuser') #=> 'testuser' will be used by rest of the specs
     def setup_test_for(adapter,user_id)
       app_id = 'application'
       s_fields = {
@@ -23,15 +25,51 @@ module Rhosync
       @ss = SourceSync.new(@s)
     end
     
+    # Executes the adapter's query method and returns 
+    # the master document (:md) stored in redis
+    # For example, if your source adapter query method was:
+    # def query(params=nil)
+    #   @result = { 
+    #     "1"=>{"name"=>"Acme", "industry"=>"Electronics"},
+    #     "2"=>{"name"=>"Best", "industry"=>"Software"}
+    #   }
+    # end
+    # 
+    # test_query would return:
+    # { 
+    #   "1"=>{"name"=>"Acme", "industry"=>"Electronics"},
+    #   "2"=>{"name"=>"Best", "industry"=>"Software"}
+    # }
     def test_query
       @ss.process_query
       return md
     end
     
+    # Returns any errors stored in redis for the previous source adapter query
+    # For example: {"query-error"=>{"message"=>"error connecting to web service!"}}
     def query_errors
       @s.get_data(:errors)
     end
     
+    # Execute's the adapter's create method with a provided record and
+    # returns the object string from the create method.  If the create method
+    # returns a string, then a link will be saved for the device next time
+    # it synchronizes.  This link can be tested here.
+    #
+    # For example, in your spec:
+    # @product = {
+    #  'name' => 'iPhone',
+    #  'brand' => 'Apple',
+    #  'price' => '$299.99',
+    #  'quantity' => '5',
+    #  'sku' => '1234'
+    # }
+    # new_product_id = test_create(@product)
+    # create_errors.should == {}
+    # md[new_product_id].should == @product
+    # 
+    # This will return the result of the adapter's create method.  The master 
+    # document (:md) should also contain the new record.
     def test_create(record)
       @c.put_data(:create,{'temp-id' => record})
       @ss.create(@c.id)
@@ -39,32 +77,72 @@ module Rhosync
       links ? links['l'] : nil
     end
     
+    # Returns any errors stored in redis from the previous source adapter create
+    # (same structure as query errors)
     def create_errors
       @c.get_data(:create_errors)
     end
     
+    # Execute the source adapter's update method.
+    # Takes a record as hash of hashes (object_id => object)
+    # 
+    # For example:
+    # test_update({'4' => {'price' => '$199.99'}})
+    # update_errors.should == {}
+    # test_query
+    # md[product_id]['price'].should == '$199.99'
+    # 
+    # This will call the adapter's update method for object_id '4'
+    # NOTE: To test the master document, you will need to run def test_query
+    # as shown above
     def test_update(record)
       @c.put_data(:update,record)
       @ss.update(@c.id)
     end
     
+    # Returns any errors stored in redis from the previous source adapter update
+    # (same structure as query errors)
     def update_errors
       @c.get_data(:update_errors)
     end
     
+    # Execute the source adapter's delete method.
+    # Takes a record as hash of hashes (object_id => object)
+    #
+    # For example:
+    # @product = {
+    #   'name' => 'iPhone',
+    #   'brand' => 'Apple',
+    #   'price' => '$299.99',
+    #   'quantity' => '5',
+    #   'sku' => '1234'
+    # }
+    # test_delete('4' => @product)
+    # delete_errors.should == {}
+    # md.should == {}
+    # 
+    # This will call the adapter's delete method for product '4'
+    # NOTE: The master document (:md) will be updated and can be
+    # verified as shown above.
     def test_delete(record)
       @c.put_data(:delete,record)
       @ss.delete(@c.id)
     end
     
+    # Returns any errors stored in redis from the previous source adapter delete
+    # (same structure as query errors)
     def delete_errors
       @c.get_data(:delete_errors)
     end
     
+    # Returns the master document (:md) for the source adapter stored in redis.
+    # This is equivalent to the @result hash of hashes structure.
     def md
       @s.get_data(:md)
     end
     
+    # Returns the client document (:cd) for the source adapter + client under test.
+    # The master document (:md) and client document (:cd) should be equal
     def cd
       @c.get_data(:cd)
     end

data/lib/rhosync/version.rb

@@ -1,3 +1,3 @@
 module Rhosync
-  VERSION = '2.0.0.beta3'
+  VERSION = '2.0.0.beta4'
 end

data/lib/rhosync.rb

@@ -122,6 +122,10 @@ module Rhosync
     settings_file = File.join(basedir,'settings','settings.yml') if basedir
     YAML.load_file(settings_file) if settings_file and File.exist?(settings_file)
   end
+  
+  def source_config
+    { "sources" => Rhosync.get_config(Rhosync.base_directory)[:sources] }
+  end
   ### End Rhosync setup methods  
   
   

data/spec/api/get_source_params_spec.rb

@@ -19,7 +19,8 @@ describe "RhosyncApiGetSourceParams" do
       {"name"=>"sync_type", "value"=>"incremental", "type"=>"string"}, 
       {"name"=>"queue", "value"=>nil, "type"=>"string"}, 
       {"name"=>"query_queue", "value"=>nil, "type"=>"string"}, 
-      {"name"=>"cud_queue", "value"=>nil, "type"=>"string"}]
+      {"name"=>"cud_queue", "value"=>nil, "type"=>"string"},
+      {"name"=>"schema", "value"=>nil, "type"=>"string"}]
   end
   
 end

data/spec/api/list_sources_spec.rb

@@ -5,13 +5,13 @@ describe "RhosyncApiListSources" do
   
   it "should list all application sources" do
     post "/api/list_sources", {:api_token => @api_token}
-    JSON.parse(last_response.body).should == ["SimpleAdapter", "SampleAdapter"]
+    JSON.parse(last_response.body).sort.should == ["SimpleAdapter", "SampleAdapter", "FixedSchemaAdapter"].sort
   end
 
   it "should list all application sources using partition_type param" do
     post "/api/list_sources", 
       {:api_token => @api_token, :partition_type => 'all'}
-    JSON.parse(last_response.body).should == ["SimpleAdapter", "SampleAdapter"]
+    JSON.parse(last_response.body).sort.should == ["SimpleAdapter", "SampleAdapter", "FixedSchemaAdapter"].sort
   end
   
   it "should list app partition sources" do
@@ -21,7 +21,7 @@ describe "RhosyncApiListSources" do
 
   it "should list user partition sources" do
     post "/api/list_sources", {:api_token => @api_token, :partition_type => :user}
-    JSON.parse(last_response.body).should == ["SampleAdapter"]
+    JSON.parse(last_response.body).sort.should == ["SampleAdapter", "FixedSchemaAdapter"].sort
   end
   
 end

data/spec/apps/rhotestapp/settings/settings.yml

@@ -4,6 +4,19 @@
   SimpleAdapter:
     poll_interval: 600
     partition_type: app
+  FixedSchemaAdapter:
+    poll_interval: 300
+    schema:
+      version: '1.0'
+      property:
+        - name: string
+        - brand: string
+        - price: string
+        - image_url: blob
+      index: 
+        - by_name_brand: 'name,brand'
+      unique_index: 
+        - by_price: 'price'
     
 :development:
   :licensefile: settings/license.key

data/spec/apps/rhotestapp/sources/fixed_schema_adapter.rb

@@ -0,0 +1,9 @@
+class FixedSchemaAdapter < SourceAdapter
+  def initialize(source,credential)
+    super(source,credential)
+  end
+  
+  def query(params=nil)
+    @result = Store.get_data('test_db_storage')
+  end
+end

data/spec/doc/doc_spec.rb

@@ -28,8 +28,6 @@ describe "Protocol" do
       :run => false,
       :secret => "secure!"
     )
-    Server.use Rack::Static, :urls => ["/data"], 
-      :root =>  File.join(File.dirname(__FILE__),'..','apps',@test_app_name)
   end
   
   before(:each) do

data/spec/jobs/bulk_data_job_spec.rb

@@ -16,16 +16,19 @@ describe "BulkDataJob" do
   it "should create sqlite data file from master document" do
     set_state('test_db_storage' => @data)
     docname = bulk_data_docname(@a.id,@u.id)
+    expected = { @s_fields[:name] => @data,
+      'FixedSchemaAdapter' => @data
+    }
     data = BulkData.create(:name => docname,
       :state => :inprogress,
       :app_id => @a.id,
       :user_id => @u.id,
-      :sources => [@s_fields[:name]])
+      :sources => [@s_fields[:name], 'FixedSchemaAdapter'])
     BulkDataJob.perform("data_name" => data.name)
     data = BulkData.load(docname)
     data.completed?.should == true
     verify_result(@s.docname(:md) => @data,@s.docname(:md_copy) => @data)
-    validate_db(data,@data).should == true
+    validate_db(data,expected).should == true
     File.exists?(data.dbfile+'.rzip').should == true
     File.exists?(data.dbfile+'.hsqldb.data').should == true
     File.exists?(data.dbfile+'.hsqldb.script').should == true
@@ -33,7 +36,8 @@ describe "BulkDataJob" do
     path = File.join(File.dirname(data.dbfile),'tmp')
     FileUtils.mkdir_p path
     unzip_file("#{data.dbfile}.rzip",path)
-    validate_db_by_name(File.join(path,File.basename(data.dbfile)),@data)
+    data.dbfile = File.join(path,File.basename(data.dbfile))
+    validate_db(data,expected).should == true
   end
   
   it "should not create hsql db files if blackberry_bulk_sync is disabled" do
@@ -49,11 +53,11 @@ describe "BulkDataJob" do
     data = BulkData.load(docname)
     data.completed?.should == true
     verify_result(@s.docname(:md) => @data,@s.docname(:md_copy) => @data)
-    validate_db(data,@data).should == true
+    validate_db(data,@s.name => @data).should == true
     File.exists?(data.dbfile+'.hsqldb.script').should == false
     File.exists?(data.dbfile+'.hsqldb.properties').should == false
   end
-  
+
   it "should create sqlite data with source metadata" do
     set_state('test_db_storage' => @data)
     mock_metadata_method([SampleAdapter]) do
@@ -69,7 +73,7 @@ describe "BulkDataJob" do
       verify_result(@s.docname(:md) => @data,
         @s.docname(:metadata) => {'foo'=>'bar'}.to_json,
         @s.docname(:md_copy) => @data)
-      validate_db(data,@data).should == true
+      validate_db(data,@s.name => @data).should == true
     end
   end
   

data/spec/server/server_spec.rb

@@ -13,6 +13,8 @@ describe "Server" do
   include Rack::Test::Methods
   include Rhosync
   
+  it_should_behave_like "DBObjectsHelper"
+  
   before(:each) do
     require File.join(get_testapp_path,@test_app_name)
     Rhosync.bootstrap(get_testapp_path) do |rhosync|
@@ -24,14 +26,12 @@ describe "Server" do
       :secret => "secure!"
     )
     Server.use Rack::Static, :urls => ["/data"], 
-      :root =>  File.join(File.dirname(__FILE__),'..','apps','rhotestapp')
+      :root =>  File.expand_path(File.join(File.dirname(__FILE__),'..','apps','rhotestapp'))
   end
 
   def app
     @app ||= Server.new
   end
-
-  it_should_behave_like "DBObjectsHelper"
   
   it "should show status page" do
     get '/'
@@ -116,8 +116,19 @@ describe "Server" do
   describe "client management routes" do
     before(:each) do
       do_post "/application/clientlogin", "login" => @u.login, "password" => 'testpass'
-      @source_config = {"sources"=>{"SampleAdapter"=>{"poll_interval"=>300},
-        "SimpleAdapter"=>{"partition_type"=>"app","poll_interval"=>600}}}
+      @source_config = {
+        "sources"=> { 
+          "FixedSchemaAdapter"=>
+            {"schema"=>{"property"=>[{"name"=>"string"}, 
+              {"brand"=>"string"}, {"price"=>"string"}, 
+              {"image_url"=>"blob"}], "version"=>"1.0", 
+              "unique_index"=>[{"by_price"=>"price"}], 
+              "index"=>[{"by_name_brand"=>"name,brand"}]}, 
+              "poll_interval"=>300},
+           "SampleAdapter"=>{"poll_interval"=>300}, 
+           "SimpleAdapter"=>{"partition_type"=>"app", 
+             "poll_interval"=>600}}
+      }
     end
     
     it "should respond to clientcreate" do
@@ -190,6 +201,20 @@ describe "Server" do
       verify_result("test_delete_storage" => {'1'=>@product1})
     end
     
+    it "should handle client posting broken json" do
+      broken_json = "{\"foo\":\"bar\"\"}"
+      post "/application", broken_json, {'CONTENT_TYPE'=>'application/json'}
+      last_response.status.should == 500
+      last_response.body.should == "Server error while processing client data"
+    end
+    
+    it "should handle client posting broken body" do
+      broken_json = ['foo']
+      post "/application", broken_json, {'CONTENT_TYPE'=>'application/json'}
+      last_response.status.should == 500
+      last_response.body.should == "Internal server error"
+    end
+    
     it "should get inserts json" do
       cs = ClientSync.new(@s,@c,1)
       data = {'1'=>@product1,'2'=>@product2}
@@ -307,7 +332,7 @@ describe "Server" do
       data = BulkData.load(bulk_data_docname(@a.id,@u.id))
       last_response.body.should == {:result => :url, 
         :url => data.url}.to_json
-      validate_db_by_name(data.dbfile,@data)
+      validate_db(data,{@s.name => @data, 'FixedSchemaAdapter' => @data})
     end
     
     it "should download bulk data file" do
@@ -318,7 +343,7 @@ describe "Server" do
       get JSON.parse(last_response.body)["url"]
       last_response.should be_ok
       File.open('test.data','wb') {|f| f.puts last_response.body}
-      validate_db_by_name('test.data',@data)
+      validate_db_file('test.data',[@s.name,'FixedSchemaAdapter'],{@s.name => @data, 'FixedSchemaAdapter' => @data})
       File.delete('test.data')
     end
   

data/spec/spec_helper.rb

@@ -58,6 +58,10 @@ module TestHelpers
   def delete_data_directory
     FileUtils.rm_rf(Rhosync.data_directory)
   end
+  
+  def json_clone(data)
+    JSON.parse(data.to_json)
+  end
       
   def set_state(state)
     state.each do |dockey,data|
@@ -97,26 +101,52 @@ module TestHelpers
   end
   
   def validate_db(bulk_data,data)
-    validate_db_by_name(bulk_data.dbfile,data)
+    validate_db_file(bulk_data.dbfile,bulk_data.sources.members,data)  
+  end
+    
+  def validate_db_file(dbfile,sources,data)  
+    db = SQLite3::Database.new(dbfile)
+    sources.each do |source_name|
+      s = Source.load(source_name,{:app_id => APP_NAME,:user_id => @u.login})
+      return false unless validate_db_by_name(db,s,data[s.name])
+    end 
+    true 
   end
   
-  def validate_db_by_name(name,data)
-    db = SQLite3::Database.new(name)
-    db.execute("select source_id,name,sync_priority,partition,sync_type,source_attribs,metadata from sources").each do |row|
-      return false if row[0] != @s.source_id.to_s
-      return false if row[1] != @s.name
-      return false if row[2] != @s.priority.to_s
-      return false if row[3] != @s.partition_type.to_s
-      return false if row[4] != @s.sync_type.to_s
-      return false if row[5] != get_attrib_counter(data)
-      return false if row[6] != @s.get_value(:metadata)
+  def validate_db_by_name(db,s,data)
+    db.execute("select source_id,name,sync_priority,partition,
+      sync_type,source_attribs,metadata from sources where name='#{s.name}'").each do |row|
+      return false if row[0] != s.source_id.to_s
+      return false if row[1] != s.name
+      return false if row[2] != s.priority.to_s
+      return false if row[3] != s.partition_type.to_s
+      return false if row[4] != s.sync_type.to_s
+      return false if row[5] != (s.schema ? "" : get_attrib_counter(data))
+      return false if row[6] != s.get_value(:metadata)
+    end
+    data = json_clone(data)
+    if s.schema
+      schema = JSON.parse(s.schema)
+      columns = ['object']
+      schema['property'].each do |column|
+        columns << column.keys[0]
+      end
+      db.execute("select #{columns.join(',')} from #{s.name}") do |row|
+        obj = data[row[0]]
+        columns.each_index do |i|
+          next if i == 0
+          return false if row[i] != obj[columns[i]]
+        end
+        data.delete(row[0])
+      end
+    else
+      db.execute("select * from object_values where source_id=#{s.source_id}").each do |row|
+        object = data[row[2]]
+        return false if object.nil? or object[row[1]] != row[3] or row[0] != s.source_id.to_s
+        object.delete(row[1])
+        data.delete(row[2]) if object.empty?
+      end
     end
-    db.execute("select * from object_values").each do |row|
-      object = data[row[2]]
-      return false if object.nil? or object[row[1]] != row[3] or row[0] != @s.source_id.to_s
-      object.delete(row[1])
-      data.delete(row[2]) if object.empty?
-    end 
     data.empty?
   end
   
@@ -232,6 +262,10 @@ describe "DBObjectsHelper", :shared => true do
     @c = Client.create(@c_fields,{:source_name => @s_fields[:name]})
     @s = Source.load(@s_fields[:name],@s_params)
     @s = Source.create(@s_fields,@s_params) if @s.nil?
+    @s1 = Source.load('FixedSchemaAdapter',@s_params)
+    @s1 = Source.create({:name => 'FixedSchemaAdapter'},@s_params) if @s1.nil?
+    config = Rhosync.source_config["sources"]['FixedSchemaAdapter']
+    @s1.update(config)
     @r = @s.read_state
     @a.sources << @s.id
     @a.users << @u.id

metadata

@@ -6,8 +6,8 @@ version: !ruby/object:Gem::Version
   - 2
   - 0
   - 0
-  - beta3
-  version: 2.0.0.beta3
+  - beta4
+  version: 2.0.0.beta4
 platform: ruby
 authors: 
 - Rhomobile
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-18 00:00:00 -07:00
+date: 2010-05-20 00:00:00 -07:00
 default_executable: rhosync
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -412,6 +412,7 @@ files:
 - spec/apps/rhotestapp/settings/license.key
 - spec/apps/rhotestapp/settings/settings.yml
 - spec/apps/rhotestapp/sources/base_adapter.rb
+- spec/apps/rhotestapp/sources/fixed_schema_adapter.rb
 - spec/apps/rhotestapp/sources/sample_adapter.rb
 - spec/apps/rhotestapp/sources/simple_adapter.rb
 - spec/apps/rhotestapp/sources/sub_adapter.rb
@@ -513,6 +514,7 @@ test_files:
 - spec/app_spec.rb
 - spec/apps/rhotestapp/application.rb
 - spec/apps/rhotestapp/sources/base_adapter.rb
+- spec/apps/rhotestapp/sources/fixed_schema_adapter.rb
 - spec/apps/rhotestapp/sources/sample_adapter.rb
 - spec/apps/rhotestapp/sources/simple_adapter.rb
 - spec/apps/rhotestapp/sources/sub_adapter.rb