rhosync 2.1.0.beta.1 → 2.1.0.beta.2

data/CHANGELOG

@@ -6,6 +6,14 @@
 * #5821277 - http stats by source not showing
 * #5899454 - move lock prefix to beginning so we don't return stats keys with it
 * #5822966 - bulk sync data file cannot handle space in the username
+* #6450519 - blob sync resend_page doesn't send metadata
+* #4646791 - cryptic error message if client exists, but source name is bogus
+* #6827511 - fill in schema column in bulk sync file sources table
+* #4490679 - support schema method in source adapter (runtime schema for bulk data)
+* #6573429 - if schema changed in any adapter, invalidate bulk data file
+* #7034095 - don't ping device if device_pin is empty or nil
+* #7089047 - fixed application.rb template store_blob method
+* #7055889 - fixed schema tables should have 'object' primary key
 
 ## 2.0.9
 * #5154725 - stats framework

data/Rakefile

@@ -18,7 +18,6 @@ begin
     :jobs   => 'spec/jobs/*_spec.rb',
     :stats => 'spec/stats/*_spec.rb',
     :ping   => 'spec/ping/*_spec.rb',
-    :doc    => 'spec/doc/*_spec.rb', 
     :generator => 'spec/generator/*_spec.rb',
     :bench => 'bench/spec/*_spec.rb'
   }
@@ -37,6 +36,12 @@ begin
     t.rcov_opts = ['--exclude', 'spec/*,gems/*,apps/*,bench/spec/*,json/*']
   end
   
+  desc "Run doc generator - dumps out doc/protocol.html"
+  Spec::Rake::SpecTask.new('doc') do |t|
+    t.spec_files = FileList['spec/doc/*_spec.rb']
+    t.rcov = false
+  end
+  
 rescue LoadError => e
   puts "rspec / rcov not available. Install with: "
   puts "gem install rspec rcov\n\n"
@@ -55,7 +60,7 @@ begin
     gemspec.homepage = %q{http://rhomobile.com/products/rhosync}
     gemspec.authors = ["Rhomobile"]
     gemspec.email = %q{dev@rhomobile.com}
-    gemspec.version = Rhosync::VERSION
+    gemspec.version = '2.1.0.beta.2'
     gemspec.files =  FileList["[A-Z]*", "{bench,bin,generators,lib,spec,tasks}/**/*"]
 
     # TODO: Due to https://www.pivotaltracker.com/story/show/3417862, we can't use JSON 1.4.3
@@ -67,7 +72,7 @@ begin
     gemspec.add_dependency "redis", "~>2.0.0"
     gemspec.add_dependency "resque", ">=1.9.7"
     gemspec.add_dependency "rest-client", ">=1.4.2"
-    gemspec.add_dependency "sinatra", "~>1.0"
+    gemspec.add_dependency "sinatra", "~>1.1"
     gemspec.add_dependency "templater", "~>1.0.0"
     gemspec.add_dependency "rake", ">=0.8.7"
     gemspec.add_development_dependency "jeweler", ">=1.4.0"

data/bench/benchapp/application.rb

@@ -18,7 +18,7 @@ class Application < Rhosync::Base
     # Override this by creating a copy of the file somewhere
     # and returning the path to that file (then don't call super!):
     # i.e. /mnt/myimages/soccer.png
-    def store_blob(blob)
+    def store_blob(obj,field_name,blob)
       super #=> returns blob[:tempfile]
     end
   end

data/examples/simple/application.rb

@@ -18,7 +18,7 @@ class Application < Rhosync::Base
     # Override this by creating a copy of the file somewhere
     # and returning the path to that file (then don't call super!):
     # i.e. /mnt/myimages/soccer.png
-    def store_blob(blob)
+    def store_blob(obj,field_name,blob)
       super #=> returns blob[:tempfile]
     end
   end

data/generators/templates/application/application.rb

@@ -18,7 +18,7 @@ class Application < Rhosync::Base
     # Override this by creating a copy of the file somewhere
     # and returning the path to that file (then don't call super!):
     # i.e. /mnt/myimages/soccer.png
-    def store_blob(blob)
+    def store_blob(object,field_name,blob)
       super #=> returns blob[:tempfile]
     end
   end

data/generators/templates/source/source_adapter.rb

@@ -40,7 +40,7 @@ class <%=class_name%> < SourceAdapter
     # TODO: write some code here if applicable
     # be sure to have a hash key and value for "object"
     # for now, we'll say that its OK to not have a delete operation
-    # raise "Please provide some code to delete a single object in the backend application using the hash values in name_value_list"
+    # raise "Please provide some code to delete a single object in the backend application using the object_id"
   end
  
   def logoff

data/lib/rhosync/client.rb

@@ -60,12 +60,15 @@ module Rhosync
     end
     
     def update_clientdoc(sources)
+      # TODO: We need to store schema info and data info in bulk data
+      # source masterdoc and source schema might have changed!
       sources.each do |source|
         s = Source.load(source,{:app_id => app_id,:user_id => user_id})
         unless s.sync_type.to_sym == :bulk_sync_only
           self.source_name = source
           Store.clone(s.docname(:md_copy),self.docname(:cd))
         end
+        self.put_value(:schema_sha1,s.get_value(:schema_sha1))
       end
     end
     

data/lib/rhosync/client_sync.rb

@@ -12,22 +12,30 @@ module Rhosync
     end
     
     def receive_cud(cud_params={},query_params=nil)
-      _process_blobs(cud_params)
-      processed = 0
-      ['create','update','delete'].each do |op|
-        key,value = op,cud_params[op]
-        processed += _receive_cud(key,value) if value
+      if @source.is_pass_through
+        @source_sync.pass_through_cud(cud_params,query_params) if value
+      else  
+        _process_blobs(cud_params)
+        processed = 0
+        ['create','update','delete'].each do |op|
+          key,value = op,cud_params[op]
+          processed += _receive_cud(key,value) if value
+        end
+        @source_sync.process_cud(@client.id) if processed > 0
       end
-      @source_sync.process_cud(@client.id) if processed > 0
     end
     
     def send_cud(token=nil,query_params=nil)
       res = []
-      if not _ack_token(token)
+      if not _ack_token(token) and not @source.is_pass_through?
         res = resend_page(token)
       else
-        @source_sync.process_query(query_params)
-        res = send_new_page
+        query_result = @source_sync.process_query(query_params)
+        if @source.is_pass_through?
+          res = send_pass_through_data(query_result)
+        else
+          res = send_new_page
+        end
       end
       _format_result(res[0],res[1],res[2],res[3])
     end
@@ -53,39 +61,85 @@ module Rhosync
     end
     
     def send_new_page
+      token,progress_count,total_count,res = '',0,0,{}
+      if schema_changed?
+        _expire_bulk_data
+        token = compute_token(@client.docname(:page_token))
+        res = {'schema-changed' => 'true'}
+      else  
+        compute_errors_page
+        res = build_page do |r|
+          progress_count,total_count,r['insert'] = compute_page
+          r['delete'] = compute_deleted_page
+          r['links'] = compute_links_page
+          r['metadata'] = compute_metadata
+        end
+        if res['insert'] or res['delete'] or res['links']
+          token = compute_token(@client.docname(:page_token))
+        else
+          _delete_errors_page 
+        end    
+        @client.put_data(:cd,res['insert'],true)      
+        @client.delete_data(:cd,res['delete'])
+      end
+      [token,progress_count,total_count,res]
+    end
+    
+    def send_pass_through_data(data)
+      data ||= {}
+      data.each_key do |object_id| 
+        data[object_id].each { |attrib,value| data[object_id][attrib] = '' if value.nil? }
+      end
+      token = ''
       compute_errors_page
-      token,progress_count,total_count = '',0,0
       res = build_page do |r|
-        progress_count,total_count,r['insert'] = compute_page
-        r['delete'] = compute_deleted_page
-        r['links'] = compute_links_page
+        r['insert'] = data
         r['metadata'] = compute_metadata
       end
-      if res['insert'] or res['delete'] or res['links']
+      if res['insert']
         token = compute_token(@client.docname(:page_token))
       else
         _delete_errors_page 
       end    
-      @client.put_data(:cd,res['insert'],true)      
-      @client.delete_data(:cd,res['delete'])
-      [token,progress_count,total_count,res]
+      [token,0,data.size,res]
     end
     
     # Resend token for a client, also sends exceptions
     def resend_page(token=nil)
-      token,progress_count,total_count = '',0,0
-      res = build_page do |r|
-        r['insert'] = @client.get_data(:page)
-        r['delete'] = @client.get_data(:delete_page)
-        r['links'] = @client.get_data(:create_links_page)
-        r['metadata'] = compute_metadata
-        progress_count = @client.get_value(:cd_size).to_i
-        total_count = @client.get_value(:total_count_page).to_i
+      token,progress_count,total_count,res = '',0,0,{}
+      schema_page = @client.get_value(:schema_page)
+      if schema_page
+        res = {'schema-changed' => 'true'}
+      else  
+        res = build_page do |r|
+          r['insert'] = @client.get_data(:page)
+          r['delete'] = @client.get_data(:delete_page)
+          r['links'] = @client.get_data(:create_links_page)
+          r['metadata'] = @client.get_value(:metadata_page)
+          progress_count = @client.get_value(:cd_size).to_i
+          total_count = @client.get_value(:total_count_page).to_i
+        end
       end
       token = @client.get_value(:page_token)
       [token,progress_count,total_count,res]
     end
     
+    # Checks if schema changed
+    def schema_changed?
+      schema_sha1 = @source.get_value(:schema_sha1)
+      
+      if @client.get_value(:schema_sha1).nil?
+        @client.put_value(:schema_sha1,schema_sha1)
+        return false
+      elsif @client.get_value(:schema_sha1) == schema_sha1
+        return false
+      end
+      
+      @client.put_value(:schema_sha1,schema_sha1)
+      @client.put_value(:schema_page,schema_sha1)
+      true
+    end
+    
     # Computes the metadata sha1 and returns metadata if client's sha1 doesn't 
     # match source's sha1
     def compute_metadata
@@ -94,9 +148,11 @@ module Rhosync
       end
       return if @client.get_value(:metadata_sha1) == metadata_sha1
       @client.put_value(:metadata_sha1,metadata_sha1)
+      @client.put_value(:metadata_page,metadata)
       metadata
     end
     
+    
     # Computes diffs between master doc and client doc, trims it to page size, 
     # stores page, and returns page as hash  
     def compute_page
@@ -215,6 +271,14 @@ module Rhosync
     end
     
     private
+    
+    # expires the bulk data for the client
+    def _expire_bulk_data
+      [:user,:app].each do |partition|
+        Rhosync.expire_bulk_data(@client.user_id,partition)
+      end
+    end
+    
     def _resend_search_result
       res = @client.get_data(:search_page)
        _format_search_result(res,res.size)
@@ -234,8 +298,8 @@ module Rhosync
     def _do_search(params={})
       # call source adapter search unless client is sending token for ack
       search_params = params[:search] if params
-      @source_sync.search(@client.id,search_params) if params.nil? or !params[:token]
-      res,diffsize = compute_search
+      res = @source_sync.search(@client.id,search_params) if params.nil? or !params[:token]
+      res,diffsize =  @source.is_pass_through? ? [res,res.size] : compute_search 
       formatted_res = _format_search_result(res,diffsize)      
       @client.flash_data('search*') if diffsize == 0
       formatted_res
@@ -250,7 +314,6 @@ module Rhosync
       else  
         search_token = @client.get_value(:search_token)
         search_token ||= ''
-        #res,diffsize = compute_search 
         return [] if res.empty?
         [ {'version'=>VERSION},
           {'token' => search_token},
@@ -285,6 +348,8 @@ module Rhosync
       if stored_token 
         if token and stored_token == token
           @client.put_value(:page_token,nil)
+          @client.flash_data(:schema_page)
+          @client.flash_data(:metadata_page)
           @client.flash_data(:create_links_page)
           @client.flash_data(:page)
           @client.flash_data(:delete_page)

data/lib/rhosync/jobs/bulk_data_job.rb

@@ -55,7 +55,7 @@ module Rhosync
       data = source.get_data(:md)
       counter = {}
       columns,qm = [],[]
-      create_table = ["\"object\" varchar"]
+      create_table = ["\"object\" varchar(255) PRIMARY KEY"]
       schema = JSON.parse(source.schema)
       
       db.transaction do |database|
@@ -84,7 +84,7 @@ module Rhosync
         schema['index'].each do |key,value|
           val2 = ""
           value.split(',').each do |col|
-            val2 += ',' if val2.length() > 0
+            val2 += ',' if val2.length > 0
             val2 += "\"#{col}\""
           end
           
@@ -95,7 +95,7 @@ module Rhosync
         schema['unique_index'].each do |key,value|
           val2 = ""
           value.split(',').each do |col|
-            val2 += ',' if val2.length() > 0
+            val2 += ',' if val2.length > 0
             val2 += "\"#{col}\""
           end
         
@@ -117,12 +117,12 @@ module Rhosync
     def self.populate_sources_table(db,sources_refs) 
       db.transaction do |database|
         database.prepare("insert into sources
-          (source_id,name,sync_priority,partition,sync_type,source_attribs,metadata,blob_attribs,associations) 
-          values (?,?,?,?,?,?,?,?,?)") do |stmt|
+          (source_id,name,sync_priority,partition,sync_type,source_attribs,metadata,schema,blob_attribs,associations) 
+          values (?,?,?,?,?,?,?,?,?,?)") do |stmt|
           sources_refs.each do |source_name,ref|
             s = ref[:source]
             stmt.execute(s.source_id,s.name,s.priority,s.partition_type,
-              s.sync_type,refs_to_s(ref[:refs]),s.get_value(:metadata),s.blob_attribs,s.has_many)
+              s.sync_type,refs_to_s(ref[:refs]),s.get_value(:metadata),s.schema,s.blob_attribs,s.has_many)
           end
         end
       end

data/lib/rhosync/jobs/ping_job.rb

@@ -11,7 +11,7 @@ module Rhosync
         client = Client.load(client_id,{:source_name => '*'})
         params.merge!('device_port' => client.device_port,
           'device_pin' => client.device_pin)   
-        if client.device_type and client.device_type.size > 0
+        if client.device_type and client.device_type.size > 0 and client.device_pin and client.device_pin.size > 0
           klass = Object.const_get(camelize(client.device_type.downcase))
           klass.ping(params) if klass
         else

data/lib/rhosync/ping/android.rb

@@ -24,6 +24,7 @@ module Rhosync
     end
     
     def self.c2d_message(params)
+      params.reject! {|k,v| v.nil? || v.length == 0}
       data = {}
       data['registration_id'] = params['device_pin']
       data['collapse_key'] = (rand * 100000000).to_i.to_s

data/lib/rhosync/server.rb

@@ -100,8 +100,11 @@ module Rhosync
         user = current_user
         if params[:source_name] and user
           @source = Source.load(params[:source_name],
-            {:user_id => user.login,:app_id => APP_NAME}) 
+            {:user_id => user.login,:app_id => APP_NAME})
+          raise "ERROR: Source '#{params[:source_name]}' requested by client doesn't exist.\n" unless @source
+          @source
         else
+          log "ERROR: Can't load source, no source_name provided.\n"
           nil
         end
       end

data/lib/rhosync/source.rb

@@ -15,7 +15,7 @@ module Rhosync
     field :queue,:string
     field :query_queue,:string
     field :cud_queue,:string
-    field :schema, :string
+    field :pass_through,:string
     attr_accessor :app_id, :user_id
     validates_presence_of :name #, :source_id
     
@@ -100,6 +100,10 @@ module Rhosync
       @app ||= App.load(self.app_id)
     end
     
+    def schema
+      @schema ||= self.get_value(:schema)
+    end
+    
     def read_state
       id = {:app_id => self.app_id,:user_id => user_by_partition,
         :source_name => self.name}
@@ -142,6 +146,10 @@ module Rhosync
       yield client_id,params if need_refresh
     end
         
+    def is_pass_through?
+      self.pass_through and self.pass_through == 'true'
+    end
+          
     private
     def self.validate_attributes(params)
       raise ArgumentError.new('Missing required attribute user_id') unless params[:user_id]

data/lib/rhosync/source_adapter.rb

@@ -62,7 +62,12 @@ module Rhosync
       @tmp_docname = @source.docname(:md) + get_random_uuid
       @stash_size = 0
       params ? self.query(params) : self.query
-      self.sync
+      if @source.is_pass_through?
+        @result
+      else
+        self.sync
+        true
+      end
     end
     
     def stash_result
@@ -73,9 +78,7 @@ module Rhosync
     end
     
     def expire_bulk_data(partition = :user)
-      name = BulkData.get_name(partition,current_user.login)
-      data = BulkData.load(name)
-      data.refresh_time = Time.now.to_i if data
+      Rhosync.expire_bulk_data(current_user.login,partition)
     end
   
     def create(name_value_list); end

data/lib/rhosync/source_sync.rb

@@ -22,6 +22,33 @@ module Rhosync
       _measure_and_process_cud('delete',client_id)
     end
     
+    # Pass through CUD to adapter, no data stored
+    def pass_through_cud(cud_params,query_params)
+      res,processed_objects = {},[]
+      begin
+        ['create','update','delete'].each do |op|
+          key,objects = op,cud_params[op]
+          objects.each do |key,value|
+            case op
+            when 'create'
+              @adapter.send(op.to_sym,value)
+            when 'update'
+              value['id'] = key
+              @adapter.send(op.to_sym,value)
+            when 'delete'
+              @adapter.send(op.to_sym,key)
+            end
+            process_objects << key
+          end if objects
+        end
+      rescue Exception => e
+        log "Error in #{op} pass through method: #{e.message}"
+        res['error'] = { 'operation' => op, 'message' => e.message } 
+      end
+      res['processed'] = process_objects
+      res.to_json
+    end
+    
     # Read Operation; params are query arguments
     def read(client_id=nil,params=nil)
       _read('query',client_id,params)
@@ -59,13 +86,15 @@ module Rhosync
     end
     
     def do_query(params=nil)
+      result = nil
       @source.if_need_refresh do
         Stats::Record.update("source:query:#{@source.name}") do
           return if _auth_op('login') == false
-          self.read(nil,params)
+          result = self.read(nil,params)
           _auth_op('logoff')
         end  
       end
+      result
     end
     
     # Enqueue a job for the source based on job type
@@ -210,12 +239,12 @@ module Rhosync
     end
     
     # Metadata Operation; source adapter returns json
-    def _get_metadata
-      if @adapter.respond_to?(:metadata)
-        metadata = @adapter.metadata 
-        if metadata
-          @source.put_value(:metadata,metadata)
-          @source.put_value(:metadata_sha1,Digest::SHA1.hexdigest(metadata))
+    def _get_data(method)
+      if @adapter.respond_to?(method)
+        data = @adapter.send(method) 
+        if data
+          @source.put_value(method,data)
+          @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
         end
       end
     end
@@ -223,17 +252,20 @@ module Rhosync
     # Read Operation; params are query arguments
     def _read(operation,client_id,params=nil)
       errordoc = nil
+      result = nil
       begin
         if operation == 'search'
           client = Client.load(client_id,{:source_name => @source.name})
           errordoc = client.docname(:search_errors)
           compute_token(client.docname(:search_token))
-          @adapter.search(params)
+          result = @adapter.search(params)
           @adapter.save(client.docname(:search))
         else
           errordoc = @source.docname(:errors)
-          _get_metadata
-          @adapter.do_query(params)
+          [:metadata,:schema].each do |method|
+            _get_data(method)
+          end  
+          result = @adapter.do_query(params)
         end
         # operation,sync succeeded, remove errors
         Store.lock(errordoc) do
@@ -246,7 +278,7 @@ module Rhosync
           Store.put_data(errordoc,{"#{operation}-error"=>{'message'=>e.message}},true)
         end
       end
-      true
+      result
     end
   end
 end

data/lib/rhosync/version.rb

@@ -1,3 +1,3 @@
 module Rhosync
-  VERSION = '2.1.0.beta.1'
-end
+  VERSION = '2.1.0'
+end

data/lib/rhosync.rb

@@ -91,7 +91,8 @@ module Rhosync
       end
       sources = config[:sources] || []
       sources.each do |source_name,fields|
-        if Source.is_exist?(source_name)  
+        check_for_schema_field!(fields)
+        if Source.is_exist?(source_name)
           s = Source.load(source_name,{:app_id => app.name,:user_id => '*'})
           s.update(fields)
         else  
@@ -143,6 +144,13 @@ module Rhosync
       log "*"*60
     end
   end
+  
+  def check_for_schema_field!(fields)
+    if fields['schema']
+      log "ERROR: 'schema' field in settings.yml is not supported anymore, please use source adapter schema method!"
+      exit(1)
+    end
+  end
 
   # Serializes oav to set element
   def setelement(obj,attrib,value)
@@ -200,6 +208,12 @@ module Rhosync
       log "*"*60
     end
   end
+  
+  def expire_bulk_data(username, partition = :user)
+    name = BulkData.get_name(partition,username)
+    data = BulkData.load(name)
+    data.refresh_time = Time.now.to_i if data
+  end
 
   def unzip_file(file_dir,params)
     uploaded_file = File.join(file_dir, params[:filename])

data/spec/api/get_source_params_spec.rb

@@ -22,7 +22,7 @@ describe "RhosyncApiGetSourceParams" do
       {"name"=>"queue", "value"=>nil, "type"=>"string"}, 
       {"name"=>"query_queue", "value"=>nil, "type"=>"string"}, 
       {"name"=>"cud_queue", "value"=>nil, "type"=>"string"},
-      {"name"=>"schema", "value"=>nil, "type"=>"string"}]
+      {"name"=>"pass_through", "value"=>nil, "type"=>"string"}]
   end
   
 end

data/spec/api/rhosync_api_spec.rb

@@ -169,7 +169,7 @@ describe "RhosyncApi" do
       {"name"=>"queue", "value"=>nil, "type"=>"string"}, 
       {"name"=>"query_queue", "value"=>nil, "type"=>"string"}, 
       {"name"=>"cud_queue", "value"=>nil, "type"=>"string"},
-      {"name"=>"schema", "value"=>nil, "type"=>"string"}]
+      {"name"=>"pass_through", "value"=>nil, "type"=>"string"}]
   end
  
   it "should list source attributes using rest call" do

data/spec/apps/rhotestapp/settings/settings.yml

@@ -9,18 +9,6 @@
     sync_type: 'incremental'
     belongs_to: 
       - brand: 'SampleAdapter'
-    schema:
-      version: '1.0'
-      property:
-        name: 'string'
-        brand: 'string'
-        price: 'string'
-        image_url_cropped: 'blob,overwrite'
-        image_url: 'blob'
-      index: 
-        by_name_brand: 'name,brand'
-      unique_index: 
-        by_price: 'price'
     
 :development:
   :licensefile: settings/license.key

data/spec/apps/rhotestapp/sources/fixed_schema_adapter.rb

@@ -6,4 +6,23 @@ class FixedSchemaAdapter < SourceAdapter
   def query(params=nil)
     @result = Store.get_data('test_db_storage')
   end
+  
+  def schema
+    {
+      'version' => '1.0',
+      'property' => {
+        'name' => 'string',
+        'brand' => 'string',
+        'price' => 'string',
+        'image_url_cropped' => 'blob,overwrite',
+        'image_url' => 'blob'
+      },
+      'index' => {
+        'by_name_brand' => 'name,brand'
+      },
+      'unique_index' => {
+        'by_price' => 'price'
+      }
+    }.to_json
+  end
 end

data/spec/client_spec.rb

@@ -61,6 +61,16 @@ describe "Client" do
       @s.docname(:md_copy) => @data)
   end
   
+  it "should update client schema_sha1" do
+    set_state(@s.docname(:md_copy) => @data, 
+      @s.docname(:schema_sha1) => 'foobar',
+      @c.docname(:cd) => {'foo' => {'bar' => 'abc'}})
+    @c.update_clientdoc([@s_fields[:name]])
+    verify_result(@c.docname(:cd) => @data,
+      @s.docname(:md_copy) => @data,
+      @c.docname(:schema_sha1) => 'foobar')  
+  end
+  
   describe "Client Stats" do
     
     before(:each) do

data/spec/client_sync_spec.rb

@@ -35,6 +35,19 @@ describe "ClientSync" do
         @cs.client.docname(:cd) => data)
     end
     
+    it "should handle send cud if pass_through is set" do
+      data = {'1'=>@product1,'2'=>@product2}
+      expected = {'insert'=>data}
+      set_test_data('test_db_storage',data)
+      @s.pass_through = 'true'
+      @cs.send_cud.should == [{'version'=>ClientSync::VERSION},
+        {'token'=>@c.get_value(:page_token)},
+        {'count'=>data.size},{'progress_count'=>data.size},
+        {'total_count'=>data.size},expected]
+      verify_result(@cs.client.docname(:page) => {},
+        @cs.client.docname(:cd) => {})
+    end
+    
     it "should return read errors in send cud" do
       msg = "Error during query"
       data = {'1'=>@product1,'2'=>@product2}
@@ -405,6 +418,113 @@ describe "ClientSync" do
       Store.get_data(@cs.client.docname(:page)).should == {}              
       @c.get_value(:page_token).should be_nil
     end
+    
+    it "should send metadata with page" do
+      expected = {'1'=>@product1}
+      set_state('test_db_storage' => expected)
+      metadata = "{\"foo\":\"bar\"}"
+      mock_metadata_method([SampleAdapter]) do
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        result.should == [{"version"=>ClientSync::VERSION},{"token"=>token}, 
+          {"count"=>1}, {"progress_count"=>0},{"total_count"=>1},
+          {'metadata'=>metadata,'insert'=>expected}]
+        @c.get_value(:metadata_page).should == metadata
+      end
+    end
+    
+    it "should send metadata with resend page" do
+      expected = {'1'=>@product1}
+      set_state('test_db_storage' => expected)
+      mock_metadata_method([SampleAdapter]) do
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        @cs.send_cud.should == [{"version"=>ClientSync::VERSION},{"token"=>token}, 
+          {"count"=>1}, {"progress_count"=>0},{"total_count"=>1},
+          {'metadata'=>"{\"foo\":\"bar\"}",'insert'=>expected}] 
+      end
+    end
+    
+    it "should ack metadata page with ack token" do
+      expected = {'1'=>@product1}
+      set_state('test_db_storage' => expected)
+      mock_metadata_method([SampleAdapter]) do
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        @cs.send_cud(token).should == [{"version"=>ClientSync::VERSION},{"token"=>""}, 
+          {"count"=>0}, {"progress_count"=>1},{"total_count"=>1},{}]
+        @c.get_value(:metadata_page).should be_nil
+      end
+    end
+    
+    it "shouldn't send schema-changed if client schema sha1 is nil" do
+      expected = {'1'=>@product1}
+      set_state('test_db_storage' => expected)
+      mock_schema_method([SampleAdapter]) do
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        result.should ==  [{"version"=>ClientSync::VERSION},{"token"=>token}, 
+          {"count"=>1}, {"progress_count"=>0},{"total_count"=>1},{'insert'=>expected}]
+        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+      end
+    end
+    
+    it "should send schema-changed instead of page" do
+      mock_schema_method([SampleAdapter]) do
+        @c.put_value(:schema_sha1,'foo')
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        result.should ==  [{"version"=>ClientSync::VERSION},{"token"=>token}, 
+          {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{'schema-changed'=>'true'}]
+        @c.get_value(:schema_page).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+      end
+    end
+    
+    it "should re-send schema-changed if no token sent" do
+      mock_schema_method([SampleAdapter]) do
+        @c.put_value(:schema_sha1,'foo')
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        @cs.send_cud.should ==  [{"version"=>ClientSync::VERSION},{"token"=>token}, 
+          {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{'schema-changed'=>'true'}]
+        @c.get_value(:schema_page).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+      end
+    end
+    
+    it "should ack schema-changed with token" do
+      mock_schema_method([SampleAdapter]) do
+        @c.put_value(:schema_sha1,'foo')
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        @cs.send_cud(token).should ==  [{"version"=>ClientSync::VERSION},{"token"=>""}, 
+          {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{}]
+        @c.get_value(:schema_page).should be_nil
+        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+      end
+    end
+    
+    it "should expire bulk data if schema changed" do
+      docname = bulk_data_docname(@a.id,@u.id)
+      data = BulkData.create(:name => docname,
+        :state => :inprogress,
+        :app_id => @a.id,
+        :user_id => @u.id,
+        :sources => [@s_fields[:name]])
+      data.refresh_time = Time.now.to_i + 600
+      mock_schema_method([SampleAdapter]) do
+        @c.put_value(:schema_sha1,'foo')
+        result = @cs.send_cud
+        token = @c.get_value(:page_token)
+        @cs.send_cud(token).should ==  [{"version"=>ClientSync::VERSION},{"token"=>""}, 
+          {"count"=>0}, {"progress_count"=>0},{"total_count"=>0},{}]
+        @c.get_value(:schema_page).should be_nil
+        @c.get_value(:schema_sha1).should == '8c148c8c1a66c7baf685c07d58bea360da87981b'
+        data = BulkData.load(docname)
+        data.refresh_time.should <= Time.now.to_i
+      end 
+    end
   end
   
   describe "bulk data" do
@@ -451,7 +571,6 @@ describe "ClientSync" do
         :app_id => @a.id,
         :user_id => name,
         :sources => [@s_fields[:name]])
-      puts "data: #{data.inspect}"
       BulkDataJob.perform("data_name" => bulk_data_docname(@a.id,name))
       data = BulkData.load(bulk_data_docname(@a.id,name))
       data.url.should match /a%20b/

data/spec/jobs/bulk_data_job_spec.rb

@@ -76,6 +76,25 @@ describe "BulkDataJob" do
     end
   end
   
+  it "should create sqlite data with source schema" do
+    set_state('test_db_storage' => @data)
+    mock_schema_method([SampleAdapter]) do
+      docname = bulk_data_docname(@a.id,@u.id)
+      data = BulkData.create(:name => docname,
+        :state => :inprogress,
+        :app_id => @a.id,
+        :user_id => @u.id,
+        :sources => [@s_fields[:name]])
+      BulkDataJob.perform("data_name" => data.name)
+      data = BulkData.load(docname)
+      data.completed?.should == true
+      verify_result(@s.docname(:md) => @data,
+        @s.docname(:schema) => "{\"property\":{\"brand\":\"string\",\"name\":\"string\"},\"version\":\"1.0\"}",
+        @s.docname(:md_copy) => @data)
+      validate_db(data,@s.name => @data).should == true
+    end
+  end
+  
   it "should raise exception if hsqldata fails" do
     data = BulkData.create(:name => bulk_data_docname(@a.id,@u.id),
       :state => :inprogress,

data/spec/jobs/ping_job_spec.rb

@@ -32,4 +32,14 @@ describe "PingJob" do
     lambda { PingJob.perform(params) }.should_not raise_error
   end
   
+  it "should skip ping for empty device_pin" do
+    params = {"user_id" => @u.id, "api_token" => @api_token,
+      "sources" => [@s.name], "message" => 'hello world', 
+      "vibrate" => '5', "badge" => '5', "sound" => 'hello.mp3'}
+    @c.device_type = 'blackberry'
+    @c.device_pin = nil
+    PingJob.should_receive(:log).once.with("Skipping ping for non-registered client_id '#{@c.id}'...")
+    lambda { PingJob.perform(params) }.should_not raise_error
+  end
+  
 end

data/spec/ping/android_spec.rb

@@ -42,4 +42,14 @@ describe "Ping Android" do
     actual['collapse_key'] = "RAND_KEY" unless actual['collapse_key'].nil?
     actual.should == expected
   end
+  
+  it "should trim empty or nil params from c2d_message" do
+    expected = {'registration_id' => @c.device_pin, 'collapse_key' => "RAND_KEY",
+      'data.vibrate' => '5', 'data.do_sync' => '', 'data.sound' => "hello.mp3"}
+    params = {"device_pin" => @c.device_pin,
+      "sources" => [], "message" => '', "vibrate" => '5', "sound" => 'hello.mp3'}
+    actual = Android.c2d_message(params)
+    actual['collapse_key'] = "RAND_KEY" unless actual['collapse_key'].nil?
+    actual.should == expected
+  end
 end

data/spec/rhosync_spec.rb

@@ -43,6 +43,18 @@ describe "Rhosync" do
     App.load(@test_app_name).sources.members.should == []
   end
   
+  it "should exit if schema config exists" do
+    config = Rhosync.get_config(get_testapp_path)
+    config[:sources]['FixedSchemaAdapter'].merge!(
+      'schema' => {'property' => 'foo'}
+    )
+    Rhosync.stub!(:get_config).and_return(config)
+    Rhosync.should_receive(:log).once.with(
+      "ERROR: 'schema' field in settings.yml is not supported anymore, please use source adapter schema method!"
+    )
+    lambda { Rhosync.bootstrap(get_testapp_path) }.should raise_error(SystemExit)
+  end
+  
   it "should add associations during bootstrap" do
     Rhosync.bootstrap(get_testapp_path)
     s = Source.load('SampleAdapter',{:app_id => @test_app_name,:user_id => '*'})

data/spec/server/server_spec.rb

@@ -12,9 +12,7 @@ describe "Server" do
 
   include Rack::Test::Methods
   include Rhosync
-  
-  it_should_behave_like "DBObjectsHelper"
-  
+    
   before(:each) do
     require File.join(get_testapp_path,@test_app_name)
     Rhosync.bootstrap(get_testapp_path) do |rhosync|
@@ -28,6 +26,8 @@ describe "Server" do
     Rhosync::Server.use Rack::Static, :urls => ["/data"], 
       :root =>  File.expand_path(File.join(File.dirname(__FILE__),'..','apps','rhotestapp'))
   end
+  
+  it_should_behave_like "DBObjectsHelper"
 
   def app
     @app ||= Rhosync::Server.new
@@ -127,17 +127,7 @@ describe "Server" do
       @source_config = {
         "sources"=>
         {"FixedSchemaAdapter"=>
-          {"schema"=>
-            {"property"=>
-              {"image_url_cropped"=>"blob,overwrite",
-               "price"=>"string",
-               "brand"=>"string",
-               "name"=>"string",
-               "image_url"=>"blob"},
-             "unique_index"=>{"by_price"=>"price"},
-             "version"=>"1.0",
-             "index"=>{"by_name_brand"=>"name,brand"}},
-           "poll_interval"=>300,
+          {"poll_interval"=>300,
            "sync_type"=>"incremental",
            "belongs_to"=>[{"brand"=>"SampleAdapter"}]},
          "SampleAdapter"=>{"poll_interval"=>300},
@@ -148,7 +138,7 @@ describe "Server" do
     it "should respond to clientcreate" do
       get "/application/clientcreate?device_type=blackberry"
       last_response.should be_ok
-      last_response.content_type.should == 'application/json'
+      last_response.content_type.should =~ /application\/json/
       id = JSON.parse(last_response.body)['client']['client_id']
       id.length.should == 32
       JSON.parse(last_response.body).should == 
@@ -235,7 +225,7 @@ describe "Server" do
       set_test_data('test_db_storage',data)
       get "/application",:client_id => @c.id,:source_name => @s.name,:version => ClientSync::VERSION
       last_response.should be_ok
-      last_response.content_type.should == 'application/json'
+      last_response.content_type.should =~ /application\/json/
       token = @c.get_value(:page_token)
       JSON.parse(last_response.body).should == [{"version"=>ClientSync::VERSION},{"token"=>token}, 
         {"count"=>2}, {"progress_count"=>0},{"total_count"=>2},{'insert'=>data}]
@@ -257,6 +247,12 @@ describe "Server" do
         {"count"=>0}, {"progress_count"=>2}, {"total_count"=>2},{}]
     end
     
+    it "should return error if source_name is unknown" do
+      get "/application",:client_id => @c.id,:source_name => 'Broken',:version => ClientSync::VERSION
+      last_response.status.should == 500
+      last_response.body.should == "ERROR: Source 'Broken' requested by client doesn't exist.\n"
+    end
+    
     it "should get deletes json" do
       cs = ClientSync.new(@s,@c,1)
       data = {'1'=>@product1,'2'=>@product2}
@@ -286,7 +282,7 @@ describe "Server" do
       params = {:client_id => @c.id,:sources => sources,:search => {'name' => 'iPhone'},
         :version => ClientSync::VERSION}
       get "/application/search",params
-      last_response.content_type.should == 'application/json'
+      last_response.content_type.should =~ /application\/json/
       token = @c.get_value(:search_token)
       JSON.parse(last_response.body).should == [[{'version'=>ClientSync::VERSION},{'token'=>token},
         {'source'=>sources[0][:name]},{'count'=>1},{'insert'=>{'1'=>@product1}}]]

data/spec/source_sync_spec.rb

@@ -72,6 +72,17 @@ describe "SourceSync" do
       end
     end
     
+    it "should process source adapter schema" do
+      mock_schema_method([SampleAdapter]) do
+        expected = {'1'=>@product1,'2'=>@product2}
+        set_state('test_db_storage' => expected)
+        @ss.process_query
+        verify_result(@s.docname(:md) => expected,
+          @s.docname(:schema) => "{\"property\":{\"brand\":\"string\",\"name\":\"string\"},\"version\":\"1.0\"}",
+          @s.docname(:schema_sha1) => "8c148c8c1a66c7baf685c07d58bea360da87981b")
+      end
+    end
+    
     it "should process source adapter with stash" do
       expected = {'1'=>@product1,'2'=>@product2}
       set_state('test_db_storage' => expected)
@@ -94,6 +105,16 @@ describe "SourceSync" do
       end
     end
     
+    it "should process source adapter with pass_through set" do
+      expected = {'1'=>@product1,'2'=>@product2}
+      set_state('test_db_storage' => expected)
+      @s.pass_through = 'true'
+      @ss.process_query.should == expected
+      verify_result(@s.docname(:md) => {},
+        @s.docname(:md_size) => nil)
+      @s.pass_through = nil
+    end
+    
     describe "create" do
       it "should do create where adapter.create returns nil" do
         set_state(@c.docname(:create) => {'2'=>@product2})
@@ -219,7 +240,7 @@ describe "SourceSync" do
         verify_result(@s.docname(:md) => expected, 
           @s.docname(:errors) => {})
       else
-        @ss.search(@c.id).should == true  
+        @ss.search(@c.id).should == expected
         verify_result(@c.docname(:search) => expected,
           @c.docname(:search_errors) => {})
       end
@@ -230,11 +251,11 @@ describe "SourceSync" do
       @ss.should_receive(:log).with("SourceAdapter raised #{operation} exception: #{msg}")
       set_test_data('test_db_storage',{},msg,"#{operation} error")
       if operation == 'query'
-        @ss.read.should == true
+        @ss.read.should be_nil
         verify_result(@s.docname(:md) => {},
           @s.docname(:errors) => {'query-error'=>{'message'=>msg}})
       else
-        @ss.search(@c.id).should == true
+        @ss.search(@c.id).should be_nil
         verify_result(@c.docname(:search) => {}, 
           @c.docname(:search_errors) => {'search-error'=>{'message'=>msg}})
       end

data/spec/spec_helper.rb

@@ -115,7 +115,7 @@ module TestHelpers
   
   def validate_db_by_name(db,s,data)
     db.execute("select source_id,name,sync_priority,partition,
-      sync_type,source_attribs,metadata,blob_attribs,associations
+      sync_type,source_attribs,metadata,schema,blob_attribs,associations
       from sources where name='#{s.name}'").each do |row|
       return false if row[0].to_s != s.source_id.to_s
       return false if row[1] != s.name
@@ -124,8 +124,9 @@ module TestHelpers
       return false if row[4] != s.sync_type.to_s
       return false if row[5] != (s.schema ? "" : get_attrib_counter(data))
       return false if row[6] != s.get_value(:metadata)
-      return false if row[7] != s.blob_attribs
-      return false if row[8] != s.has_many
+      return false if row[7] != s.schema
+      return false if row[8] != s.blob_attribs
+      return false if row[9] != s.has_many
     end
 
     data = json_clone(data)
@@ -174,6 +175,24 @@ module TestHelpers
     end
   end
   
+  def mock_schema_method(adapters, &block)
+    adapters.each do |klass|
+      klass.class_eval 'def schema
+        {
+          "property" => {
+            "name" => "string",
+            "brand" => "string"
+          },
+          "version" => "1.0"
+        }.to_json
+      end'
+    end
+    yield
+    adapters.each do |klass|
+      klass.class_eval "def schema; end"
+    end
+  end
+  
   def unzip_file(file,file_dir)
     Zip::ZipFile.open(file) do |zip_file|
       zip_file.each do |f|

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: rhosync
 version: !ruby/object:Gem::Version 
-  hash: 62196465
+  hash: 62196471
   prerelease: true
   segments: 
   - 2
   - 1
   - 0
   - beta
-  - 1
-  version: 2.1.0.beta.1
+  - 2
+  version: 2.1.0.beta.2
 platform: ruby
 authors: 
 - Rhomobile
@@ -17,7 +17,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-07 00:00:00 -07:00
+date: 2010-12-07 00:00:00 -08:00
 default_executable: rhosync
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -156,11 +156,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 15
+        hash: 13
         segments: 
         - 1
-        - 0
-        version: "1.0"
+        - 1
+        version: "1.1"
   type: :runtime
   version_requirements: *id009
 - !ruby/object:Gem::Dependency 
@@ -551,8 +551,8 @@ homepage: http://rhomobile.com/products/rhosync
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -583,6 +583,9 @@ signing_key:
 specification_version: 3
 summary: RhoSync Synchronization Framework
 test_files: 
+- examples/simple/application.rb
+- examples/simple/sources/sample_adapter.rb
+- examples/simple/sources/simple_adapter.rb
 - spec/api/api_helper.rb
 - spec/api/create_client_spec.rb
 - spec/api/create_user_spec.rb
@@ -616,8 +619,8 @@ test_files:
 - spec/apps/rhotestapp/sources/sample_adapter.rb
 - spec/apps/rhotestapp/sources/simple_adapter.rb
 - spec/apps/rhotestapp/sources/sub_adapter.rb
-- spec/apps/rhotestapp/vendor/mygem-0.1.0/lib/mygem/mygem.rb
 - spec/apps/rhotestapp/vendor/mygem-0.1.0/lib/mygem.rb
+- spec/apps/rhotestapp/vendor/mygem-0.1.0/lib/mygem/mygem.rb
 - spec/bulk_data/bulk_data_spec.rb
 - spec/client_spec.rb
 - spec/client_sync_spec.rb
@@ -649,6 +652,3 @@ test_files:
 - spec/sync_states_spec.rb
 - spec/test_methods_spec.rb
 - spec/user_spec.rb
-- examples/simple/application.rb
-- examples/simple/sources/sample_adapter.rb
-- examples/simple/sources/simple_adapter.rb