rhosync 2.0.0.beta2 → 2.0.0.beta3

data/generators/rhosync.rb

@@ -59,6 +59,11 @@ module Rhosync
       template.source = 'Rakefile'
       template.destination = "#{name}/Rakefile"
     end
+    
+    template :spec_helper do |template|
+      template.source = 'spec/spec_helper.rb'
+      template.destination = "#{name}/spec/spec_helper.rb"
+    end
   end
   
   class SourceGenerator < BaseGenerator
@@ -78,7 +83,7 @@ module Rhosync
     template :source do |template|
       template.source = 'source_adapter.rb'
       template.destination = "sources/#{underscore_name}.rb"
-      settings_file = 'settings/settings.yml'
+      settings_file = File.join(@destination_root,'settings','settings.yml')
       settings = YAML.load_file(settings_file)
       settings[:sources] ||= {}
       settings[:sources][class_name] = {:poll_interval => 300}
@@ -91,6 +96,11 @@ module Rhosync
         file.write envs.to_yaml[3..-1]
       end
     end
+    
+    template :source_spec do |template|
+      template.source = 'source_spec.rb'
+      template.destination = "spec/sources/#{underscore_name}_spec.rb"
+    end
   end
   
   add :app, AppGenerator

data/generators/templates/application/spec/spec_helper.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+
+# Set environment to test
+ENV['RHO_ENV'] = 'test'
+ROOT_PATH = ENV['ROOT_PATH']
+
+# Try to load vendor-ed rhosync, otherwise load the gem
+begin
+  require 'vendor/rhosync/lib/rhosync'
+rescue LoadError
+  require 'rhosync'
+end
+
+# Load our rhosync application
+require 'application'
+include Rhosync
+
+require 'rhosync/test_methods'
+
+describe "SpecHelper", :shared => true do
+  include Rhosync::TestMethods
+  
+  before(:each) do
+    Store.db.flushdb
+    Application.initializer(ROOT_PATH)
+  end
+end

data/generators/templates/source/source_adapter.rb

@@ -7,7 +7,7 @@ class <%=class_name%> < SourceAdapter
     # TODO: Login to your data source here if necessary
   end
  
-  def query
+  def query(params=nil)
     # TODO: Query your backend data source and assign the records 
     # to a nested hash structure called @result. For example:
     # @result = { 

data/generators/templates/source/source_spec.rb

@@ -0,0 +1,25 @@
+require File.join(File.dirname(__FILE__),'..','spec_helper')
+
+describe "<%=class_name%>" do
+  it_should_behave_like "SpecHelper"
+  
+  before(:each) do
+    setup_test_for <%=class_name%>,'testuser'
+  end
+  
+  it "should process <%=class_name%> query" do
+    pending
+  end
+  
+  it "should process <%=class_name%> create" do
+    pending
+  end
+  
+  it "should process <%=class_name%> update" do
+    pending
+  end
+  
+  it "should process <%=class_name%> delete" do
+    pending
+  end
+end

data/lib/rhosync/app.rb

@@ -50,16 +50,14 @@ module Rhosync
     
     def partition_sources(partition,user_id)
       names = []
-      need_refresh = false
       sources.members.each do |source|
         s = Source.load(source,{:app_id => self.name,
           :user_id => user_id})
         if s.partition == partition
           names << s.name
-          need_refresh = true if !need_refresh and s.check_refresh_time  
         end
       end
-      {:names => names,:need_refresh => need_refresh}
+      names
     end
     
     def store_blob(obj,field_name,blob)

data/lib/rhosync/bulk_data/bulk_data.rb

@@ -39,7 +39,7 @@ module Rhosync
     end
     
     def url
-      dbfile
+      File.join('/data',dbfile[Rhosync.data_directory.length..-1])
     end
     
     class << self

data/lib/rhosync/bulk_data/syncdb.schema

@@ -5,14 +5,12 @@ CREATE TABLE client_info (
  token_sent BIGINT default 0,
  reset BIGINT default 0,
  port VARCHAR(10) default NULL,
- bulksync_state BIGINT default 0,
  last_sync_success VARCHAR(100) default NULL);
 CREATE TABLE object_values (
  source_id BIGINT default NULL,
  attrib varchar(255) default NULL,
  object varchar(255) default NULL,
- value varchar default NULL,
- attrib_type varchar(255) default NULL);
+ value varchar default NULL);
 CREATE TABLE changed_values (
  source_id BIGINT default NULL,
  attrib varchar(255) default NULL,
@@ -25,13 +23,18 @@ CREATE TABLE sources (
  source_id BIGINT PRIMARY KEY,
  name VARCHAR(255) default NULL,
  token BIGINT default NULL,
- priority BIGINT,
+ sync_priority BIGINT,
  partition VARCHAR(255),
  sync_type VARCHAR(255),
+ metadata varchar default NULL, 
  last_updated BIGINT default 0,
  last_inserted_size BIGINT default 0,
  last_deleted_size BIGINT default 0,
  last_sync_duration BIGINT default 0,
  last_sync_success BIGINT default 0,
  backend_refresh_time BIGINT default 0,
- source_attribs varchar default NULL);
+ source_attribs varchar default NULL,
+ schema varchar default NULL,
+ schema_version varchar default NULL,
+ associations varchar default NULL,
+ blob_attribs varchar default NULL);

data/lib/rhosync/client_sync.rb

@@ -173,17 +173,18 @@ module Rhosync
         name = BulkData.get_name(partition,client)
         data = BulkData.load(name)
         sources = client.app.partition_sources(partition,client.user_id)
-        if (data.nil? or (data.completed? and sources[:need_refresh] == true)) and 
-          sources[:names].length > 0 
+        if (data.nil? or (data.completed? and !File.exist?(data.dbfile)) or 
+          (data.completed? and data.refresh_time <= Time.now.to_i)) and sources.length > 0 
           data.delete if data
           data = BulkData.create(:name => name,
             :app_id => client.app_id,
             :user_id => client.user_id,
-            :sources => sources[:names])
+            :sources => sources,
+            :refresh_time => Time.now.to_i + Rhosync.bulk_sync_poll_interval)
           BulkData.enqueue("data_name" => name)
         end
         if data and data.completed? 
-          client.update_clientdoc(sources[:names])
+          client.update_clientdoc(sources)
           {:result => :url, :url => data.url}
         elsif data
           {:result => :wait}

data/lib/rhosync/jobs/bulk_data_job.rb

@@ -1,4 +1,5 @@
 require 'sqlite3'
+require 'zip/zip'
 
 module Rhosync
   module BulkDataJob
@@ -16,13 +17,17 @@ module Rhosync
           create_sqlite_data_file(bulk_data,ts)
           timer = lap_timer('create_sqlite_data_file',timer)
           create_hsql_data_file(bulk_data,ts) if Rhosync.blackberry_bulk_sync
-          timer = lap_timer('create_hsql_data_file',timer)
+          lap_timer('create_hsql_data_file',timer)
+          log "finished bulk data process"
           bulk_data.state = :completed
+          bulk_data.refresh_time = Time.now.to_i + Rhosync.bulk_sync_poll_interval
         else
           raise Exception.new("No bulk data found for #{params["data_name"]}")
         end
       rescue Exception => e
         bulk_data.delete if bulk_data
+        log "Bulk data job raised: #{e.message}"
+        log e.backtrace.join("\n")
         raise e
       end
     end
@@ -55,12 +60,12 @@ module Rhosync
     def self.populate_sources_table(db,sources_refs) 
       db.transaction do |database|
         database.prepare("insert into sources
-          (source_id,name,priority,partition,sync_type,source_attribs) 
-          values (?,?,?,?,?,?)") do |stmt|
+          (source_id,name,sync_priority,partition,sync_type,source_attribs,metadata) 
+          values (?,?,?,?,?,?,?)") do |stmt|
           sources_refs.each do |source_name,ref|
             s = ref[:source]
             stmt.execute(s.source_id,s.name,s.priority,s.partition_type,
-              s.sync_type,refs_to_s(ref[:refs]))
+              s.sync_type,refs_to_s(ref[:refs]),s.get_value(:metadata))
           end
         end
       end
@@ -74,6 +79,7 @@ module Rhosync
       db.execute_batch(File.open(schema,'r').read)
       src_counter = 1
       bulk_data.sources.members.sort.each do |source_name|
+        timer = start_timer("start importing sqlite data for #{source_name}")
         source = Source.load(source_name,{:app_id => bulk_data.app_id,
           :user_id => bulk_data.user_id})
         source.source_id = src_counter
@@ -81,9 +87,11 @@ module Rhosync
         source_attrib_refs = import_data_to_object_values(db,source)
         sources_refs[source_name] = 
           {:source => source, :refs => source_attrib_refs}
+        lap_timer("finished importing sqlite data for #{source_name}",timer)
       end
       populate_sources_table(db,sources_refs)
       db.execute_batch(File.open(index,'r').read)
+      compress("#{bulk_data.dbfile}.rzip",bulk_data.dbfile)
     end
     
     def self.create_hsql_data_file(bulk_data,ts)
@@ -100,5 +108,11 @@ module Rhosync
       dbfile = File.join(Rhosync.data_directory,bulk_data_name+'_'+ts+'.data')
       [schema,index,dbfile]
     end
+    
+    def self.compress(archive,file)
+      Zip::ZipFile.open(archive, 'w') do |zipfile|
+        zipfile.add(File.basename(file),file)
+      end
+    end
   end
 end

data/lib/rhosync/server.rb

@@ -44,7 +44,13 @@ module Rhosync
       end
 
       def do_login
-        login ? status(200) : status(401)
+        begin
+          login ? status(200) : status(401)
+        rescue LoginException => le
+          throw :halt, [401, le.message]
+        rescue Exception => e
+          throw :halt, [500, e.message]
+        end
       end
 
       def login_required
@@ -52,10 +58,10 @@ module Rhosync
       end
 
       def login
-        if current_app and current_app.can_authenticate?
-          user = current_app.authenticate(params[:login], params[:password], session)
-        else
+        if params[:login] == 'admin'
           user = User.authenticate(params[:login], params[:password])
+        elsif current_app and current_app.can_authenticate?
+          user = current_app.authenticate(params[:login], params[:password], session)
         end
         if user
           session[:login] = user.login

data/lib/rhosync/source_adapter.rb

@@ -40,21 +40,37 @@ module Rhosync
     def search(params=nil); end
     
     def sync
-      return if _result_nil?
-      if @result.empty?
+      if @result and @result.empty?
         @source.lock(:md) do |s|
           s.flash_data(:md)
           s.put_value(:md_size,0)
         end
       else
-        tmp_docname = @source.docname(:md) + get_random_uuid
-        Store.put_data(tmp_docname,@result)
+        if @result
+          Store.put_data(@tmp_docname,@result) 
+          @stash_size += @result.size
+        end  
         @source.lock(:md) do |s|
-          Store.rename(tmp_docname,s.docname(:md))
-          s.put_value(:md_size,@result.size)
+          s.flash_data(:md)
+          Store.rename(@tmp_docname,s.docname(:md))
+          s.put_value(:md_size,@stash_size)
         end
       end
     end
+    
+    def do_query(params=nil)
+      @tmp_docname = @source.docname(:md) + get_random_uuid
+      @stash_size = 0
+      params ? self.query(params) : self.query
+      self.sync
+    end
+    
+    def stash_result
+      return if @result.nil?
+      Store.put_data(@tmp_docname,@result,true)
+      @stash_size += @result.size
+      @result = nil
+    end
   
     def create(name_value_list); end
 
@@ -67,29 +83,17 @@ module Rhosync
     def logoff; end
     
     def save(docname)
-      return if _result_nil?
+      return if @result.nil?
       if @result.empty?
         Store.flash_data(docname)
       else
         Store.put_data(docname,@result)
       end
     end
-  
-    # only implement this if you want RhoSync to install a callback into your backend
-    # def set_callback(notify_url)
-    # end
-  
-    MSG_NIL_RESULT_ATTRIB = "You might have expected a synchronization but the @result attribute was 'nil'"
     
     protected
     def current_user
       @source.user
     end
-  
-    private 
-    def _result_nil? #:nodoc:
-      log MSG_NIL_RESULT_ATTRIB if @result.nil?
-      @result.nil?
-    end
   end
 end

data/lib/rhosync/source_sync.rb

@@ -225,8 +225,7 @@ module Rhosync
         else
           errordoc = @source.docname(:errors)
           _get_metadata
-          params ? @adapter.query(params) : @adapter.query
-          @adapter.sync
+          @adapter.do_query(params)
         end
         # operation,sync succeeded, remove errors
         Store.lock(errordoc) do

data/lib/rhosync/store.rb

@@ -12,7 +12,8 @@ module Rhosync
       
       def create(server=nil)
         @@db ||= _get_redis(server)
-        raise "Error connecting to Redis store." unless @@db and (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client))
+        raise "Error connecting to Redis store." unless @@db and 
+          (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client) or @@db.is_a?(Redis::Distributed))
       end
   
       # Adds set with given data, replaces existing set
@@ -53,7 +54,7 @@ module Rhosync
     
       # Retrieves value for a given key
       def get_value(dockey)
-          @@db.get(dockey) if dockey
+        @@db.get(dockey) if dockey
       end
   
       # Retrieves set for given dockey,source,user
@@ -182,6 +183,8 @@ module Rhosync
           host,port,db,password = server.split(':')
           Redis.new(:thread_safe => true, :host => host,
             :port => port, :db => db, :password => password)
+        elsif server and server.is_a?(Array)
+          Redis::Distributed.new :hosts => server
         else
           Redis.new(:thread_safe => true)
         end

data/lib/rhosync/tasks.rb

@@ -102,13 +102,20 @@ namespace :rhosync do
   task :create_user => :config do
     login = ask "new user login: "
     password = ask "new user password: "
-    RhosyncApi.create_user($url,$appname,$token,login,password)
+    RhosyncApi.create_user($url,$token,login,password)
   end
   
-  desc "Deletes the user from rhosync"
+  desc "Deletes a user from rhosync"
   task :delete_user => :config do
     login = ask "user to delete: "
-    RhosyncApi.delete_user($url,$appname,$token,login)
+    RhosyncApi.delete_user($url,$token,login)
+  end
+  
+  desc "Deletes a device from rhosync"
+  task :delete_device => :config do
+    user_id = ask "device's user_id: "
+    device_id = ask "device to delete: "
+    RhosyncApi.delete_client($url,$token,user_id,device_id)
   end
   
   # desc "Updates an existing user in rhosync"
@@ -120,22 +127,37 @@ namespace :rhosync do
   #     :login => login, :password => password, :attributes => {:new_password => new_password}})
   # end
   
-  # desc "Reset source refresh time"
-  # task :reset_refresh_time => :config do
-  #   user = ask "user: "
-  #   source_name = ask "source name: "
-  #   post("/api/set_refresh_time", {:api_token => $token, :app_name => $appname,
-  #     :user_name => user, :source_name => source_name})
-  # end
+  desc "Reset source refresh time"
+  task :reset_refresh => :config do
+    user = ask "user: "
+    source_name = ask "source name: "
+    RestClient.post("#{$url}/api/set_refresh_time", {:api_token => $token,
+      :user_name => user, :source_name => source_name})
+  end
   
-  desc "Run rhosync source adapter specs"
-  task :spec do
-    files = File.join($app_basedir,'rhosync/spec/sources/*_spec.rb')
-    Spec::Rake::SpecTask.new('rhosync:spec') do |t|
-      t.spec_files = FileList[files]
-      t.spec_opts = %w(-fs --color)
-      t.rcov = true
-      t.rcov_opts = ['--exclude', 'spec/*,gems/*']
+  begin
+    require 'spec/rake/spectask'
+    require 'rcov/rcovtask' unless windows?
+
+    desc "Run source adapter specs"
+    task :spec do
+      files = File.join('spec','sources','*_spec.rb')
+      Spec::Rake::SpecTask.new('rhosync:spec') do |t|
+        t.spec_files = FileList[files]
+        t.spec_opts = %w(-fn -b --color)
+        unless windows?
+          t.rcov = true
+          t.rcov_opts = ['--exclude', 'spec/*,gems/*']
+        end
+      end
+    end
+  rescue LoadError
+    if windows?
+      puts "rspec not available. Install it with: "
+      puts "gem install rspec\n\n"
+    else
+      puts "rspec / rcov not available. Install it with: "
+      puts "gem install rspec rcov\n\n"
     end
   end
   
@@ -160,6 +182,11 @@ namespace :rhosync do
   task :attach => :dtach_installed do
     sh "dtach -a #{rhosync_socket}" unless windows?
   end
+  
+  desc "Launch the web console in a browser - uses :syncserver: in settings.yml"
+  task :web => :config do
+    windows? ? sh("start #{$url}") : sh("open #{$url}")
+  end
 end
 
 load File.join(File.dirname(__FILE__),'..','..','tasks','redis.rake')

data/lib/rhosync/test_methods.rb

@@ -0,0 +1,72 @@
+module Rhosync
+  class SourceAdapter
+    attr_accessor :result
+  end
+  
+  module TestMethods
+    def setup_test_for(adapter,user_id)
+      app_id = 'application'
+      s_fields = {
+        :user_id => user_id,
+        :app_id => app_id
+      }
+      c_fields = {
+        :device_type => 'iPhone',
+        :device_pin => 'abcd',
+        :device_port => '3333',
+        :user_id => user_id,
+        :app_id => app_id 
+      }
+      @u = User.create(:login => user_id)
+      @s = Source.load(adapter.to_s,s_fields)
+      @c = Client.create(c_fields,{:source_name => adapter.to_s})
+      @ss = SourceSync.new(@s)
+    end
+    
+    def test_query
+      @ss.process_query
+      return md
+    end
+    
+    def query_errors
+      @s.get_data(:errors)
+    end
+    
+    def test_create(record)
+      @c.put_data(:create,{'temp-id' => record})
+      @ss.create(@c.id)
+      links = @c.get_data(:create_links)['temp-id']
+      links ? links['l'] : nil
+    end
+    
+    def create_errors
+      @c.get_data(:create_errors)
+    end
+    
+    def test_update(record)
+      @c.put_data(:update,record)
+      @ss.update(@c.id)
+    end
+    
+    def update_errors
+      @c.get_data(:update_errors)
+    end
+    
+    def test_delete(record)
+      @c.put_data(:delete,record)
+      @ss.delete(@c.id)
+    end
+    
+    def delete_errors
+      @c.get_data(:delete_errors)
+    end
+    
+    def md
+      @s.get_data(:md)
+    end
+    
+    def cd
+      @c.get_data(:cd)
+    end
+  end
+end

data/lib/rhosync/version.rb

@@ -1,3 +1,3 @@
 module Rhosync
-  VERSION = '2.0.0.beta2'
+  VERSION = '2.0.0.beta3'
 end

data/lib/rhosync.rb

@@ -1,4 +1,5 @@
 require 'redis'
+require 'redis/distributed'
 require 'json'
 require 'base64'
 require 'zip/zip'
@@ -29,12 +30,16 @@ module Rhosync
   
   class InvalidArgumentError < RuntimeError; end
   class RhosyncServerError < RuntimeError; end
+  
+  # Used by application authenticate to indicate login failure
+  class LoginException < RuntimeError; end
+  
   extend self
     
   class << self
     attr_accessor :base_directory, :app_directory, :data_directory, 
       :vendor_directory, :blackberry_bulk_sync, :redis, :environment,
-      :log_disabled, :license
+      :log_disabled, :license, :bulk_sync_poll_interval
   end
   
   ### Begin Rhosync setup methods  
@@ -49,6 +54,7 @@ module Rhosync
     Rhosync.data_directory = get_setting(config,environment,:data_directory)
     Rhosync.vendor_directory = get_setting(config,environment,:vendor_directory)
     Rhosync.blackberry_bulk_sync = get_setting(config,environment,:blackberry_bulk_sync,false)
+    Rhosync.bulk_sync_poll_interval = get_setting(config,environment,:bulk_sync_poll_interval,3600)
     Rhosync.redis = get_setting(config,environment,:redis,false)
     Rhosync.log_disabled = get_setting(config,environment,:log_disabled,false)
     Rhosync.environment = environment    
@@ -60,6 +66,7 @@ module Rhosync
     Rhosync.data_directory ||= File.join(Rhosync.base_directory,'data')
     Rhosync.vendor_directory ||= File.join(Rhosync.base_directory,'vendor')
     Rhosync.blackberry_bulk_sync ||= false
+    Rhosync.bulk_sync_poll_interval ||= 3600
     Rhosync.log_disabled ||= false
     Rhosync.license = License.new
     
@@ -113,7 +120,7 @@ module Rhosync
   def get_config(basedir)
     # Load settings
     settings_file = File.join(basedir,'settings','settings.yml') if basedir
-    config = YAML.load_file(settings_file) if settings_file and File.exist?(settings_file)
+    YAML.load_file(settings_file) if settings_file and File.exist?(settings_file)
   end
   ### End Rhosync setup methods  
   
@@ -129,13 +136,15 @@ module Rhosync
 
   # Serializes oav to set element
   def setelement(obj,attrib,value)
-    "#{obj}:#{attrib}:#{Base64.encode64(value.to_s)}"
+    #"#{obj}:#{attrib}:#{Base64.encode64(value.to_s)}"
+    "#{obj}:#{attrib}:#{value.to_s}"
   end
 
   # De-serializes oav from set element
   def getelement(element)
-    res = element.split(':')
-    [res[0], res[1], Base64.decode64(res[2].to_s)]
+    res = element.split(':',3)
+    #[res[0], res[1], Base64.decode64(res[2].to_s)]
+    [res[0], res[1], res[2]]
   end
 
   # Get random UUID string
@@ -224,7 +233,7 @@ module Rhosync
   end
   
   # Base rhosync application class
-  class Base
+  class Base    
     # Add everything in vendor to load path
     # TODO: Integrate with 3rd party dependency management
     def self.initializer(path=nil)

data/spec/apps/rhotestapp/application.rb

@@ -2,7 +2,9 @@ class Application < Rhosync::Base
   class << self
     def authenticate(username,password,session)
       session[:auth] = "delegated"
-      password == 'wrongpass' ? false : true
+      raise RuntimeError.new('server error') if password == 'server error'
+      raise LoginException.new('login exception') if password == 'wrongpass'
+      password == 'wrongpassnomsg' ? false : true
     end
     
     # Add hooks for application startup here

data/spec/apps/rhotestapp/settings/settings.yml

@@ -11,13 +11,16 @@
   :iphonepassphrase: certpassword
   :iphoneserver: gateway.sandbox.push.apple.com
   :iphoneport: 2195
+  :bulk_sync_poll_interval: 3600
   :redis: localhost:6379
   :syncserver: http://localhost:9292/application/
 :test: 
   :licensefile: settings/license.key
+  :bulk_sync_poll_interval: 3600
   :redis: localhost:6379
   :syncserver: http://localhost:9292/application/
 :production: 
   :licensefile: settings/license.key
+  :bulk_sync_poll_interval: 3600
   :redis: localhost:6379
   :syncserver: http://localhost:9292/application/