rhosync 2.0.0.beta1

data/README.md

@@ -0,0 +1,26 @@
+Rhosync
+-------------------------------------------------------------
+
+Redis-powered rhosync with built-in sinatra application.
+
+INSTALL
+-------------------------------------------------------------
+1. Make sure you have the following gems installed:
+
+	* rspec rcov json sqlite3-ruby faker redis redis-namespace sinatra rack-test rubyzip uuidtools resque
+	
+2. Install and start a redis server (v1.2 or greater is required) (see <http://code.google.com/p/redis/>)
+
+3. Install hsqldata.jar to vendor/ directory.  See <http://github.com/rhomobile/hsqldata> for instructions on how to build hsqldata.
+
+4. "rake" to run all specs
+
+Windows Notes: when run any spec task error message box (ruby.exe - Unable to locate component) will appear. Just press 'OK'. This is problem with rcov.
+
+DOCS
+-------------------------------------------------------------
+  * Intro to Rhodes & Rhosync: <http://wiki.rhomobile.com/index.php/Rhomobile> -> Read this first!
+  * Tutorial:                  <http://wiki.rhomobile.com/index.php/Tutorial> 
+  * Architecture:              <http://wiki.rhomobile.com/index.php/RhoSync_2.0>
+  * Rdoc (still rough):        <http://rdoc.info/projects/rhomobile/rhosync>
+  * Client/Server Protocol:    See doc/protocol.html

data/Rakefile

@@ -0,0 +1,109 @@
+require 'yaml'
+
+$:.unshift File.join(File.dirname(__FILE__),'lib')
+require 'rhosync'
+
+task :default => 'spec:all'
+task :spec => 'spec:spec'
+
+begin
+  require 'spec/rake/spectask'
+  require 'rcov/rcovtask'
+  
+  SPEC_OPTS = ['-fn', '--color', '-b']
+         
+  TYPES = { 
+    :spec   => 'spec/*_spec.rb',
+    :perf   => 'spec/perf/*_spec.rb',
+    :server => 'spec/server/*_spec.rb',
+    :api    => 'spec/api/*_spec.rb',
+    :bulk   => 'spec/bulk_data/*_spec.rb',
+    :jobs   => 'spec/jobs/*_spec.rb',
+    :ping   => 'spec/ping/*_spec.rb',
+    :doc    => 'spec/doc/*_spec.rb', 
+    :generator => 'spec/generator/*_spec.rb',
+    :bench => 'bench/spec/*_spec.rb'
+  }
+ 
+  TYPES.each do |type,files|
+    desc "Run specs in #{files}"
+    Spec::Rake::SpecTask.new("spec:#{type}") do |t|
+      t.spec_files = FileList[TYPES[type]]
+      t.spec_opts = SPEC_OPTS
+    end
+  end
+
+  desc "Run specs in spec/**/*_spec.rb "
+  Spec::Rake::SpecTask.new('spec:all') do |t|
+    t.spec_files = FileList[TYPES.values]
+    t.spec_opts = SPEC_OPTS
+    t.rcov = true
+    t.rcov_opts = ['--exclude', 'spec/*,gems/*,apps/*,bench/spec/*']
+  end
+  
+rescue LoadError => e
+  puts "rspec / rcov not available. Install with: "
+  puts "gem install rspec rcov\n\n"
+end
+
+desc "Build rhosync gem"
+task :gem => [ 'spec:all', 'clobber_spec:all', :gemspec, :build ]
+
+begin
+  require 'jeweler'
+
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "rhosync"
+    gemspec.summary = %q{Rhosync Server}
+    gemspec.description = %q{Rhosync Server and related command-line utilities for using Rhosync}
+    gemspec.homepage = %q{http://rhomobile.com/products/rhosync}
+    gemspec.authors = ["Rhomobile"]
+    gemspec.version = Rhosync::VERSION
+    gemspec.files =  FileList["[A-Z]*", "{bench,bin,doc,generators,lib,spec,tasks}/**/*"]
+
+    gemspec.add_dependency "json", ">=1.2.3"
+    gemspec.add_dependency "log4r", ">=1.1.7"
+    gemspec.add_dependency "sqlite3-ruby", ">=1.2.5"
+    gemspec.add_dependency "rubyzip", ">=0.9.4"
+    gemspec.add_dependency "uuidtools", ">=2.1.1"
+    gemspec.add_dependency "redis", ">=0.2.0"
+    gemspec.add_dependency "resque", ">=1.8.0"
+    gemspec.add_dependency "rest-client", ">=1.4.2"
+    gemspec.add_dependency "sinatra", ">=0.9.2"
+    gemspec.add_dependency "templater", ">=1.0.0"
+    gemspec.add_development_dependency "jeweler", ">=1.4.0"
+    gemspec.add_development_dependency "rspec", ">=1.3.0"
+    gemspec.add_development_dependency "rcov", ">=0.9.8"
+    gemspec.add_development_dependency "faker", ">=0.3.1"
+    gemspec.add_development_dependency "rack-test", ">=0.5.3"
+    gemspec.add_development_dependency "mechanize", ">=1.0.0"
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: "
+  puts "gem install jeweler\n\n"
+end
+
+desc "Load console environment"
+task :console do
+  sh "irb -rubygems -r #{File.join(File.dirname(__FILE__),'lib','rhosync','server.rb')}"
+end
+
+desc "Run benchmark scripts"
+task :bench do
+  login = ask "login: "
+  password = ask "password: "
+  prefix = 'bench/scripts/'
+  suffix = '_script.rb'
+  list = ask "scripts(default is '*'): "
+  file_list = list.empty? ? FileList[prefix+'*'+suffix] : FileList[prefix+list+suffix]
+  file_list.each do |script|
+    sh "bench/bench start #{script} #{login} #{password}"
+  end
+end
+
+def ask(msg)
+  print msg
+  STDIN.gets.chomp
+end
+
+load 'tasks/redis.rake'

data/bench/bench

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require File.join(File.dirname(__FILE__),'lib','bench')
+
+Bench::Cli.start

data/bench/benchapp/Rakefile

@@ -0,0 +1,14 @@
+$:.unshift File.join(File.dirname(__FILE__),'..','..','lib')
+require 'rhosync/tasks'
+
+begin
+  require 'resque/tasks'
+
+  task "resque:setup" do
+    require 'rhosync'
+    require 'application'
+  end
+rescue LoadError
+  puts "Resque not available. Install it with: "
+  puts "gem install resque\n\n"
+end

data/bench/benchapp/application.rb

@@ -0,0 +1,13 @@
+class Application < Rhosync::Base
+  class << self
+    def authenticate(username,password,session)
+      true # do some interesting authentication here...
+    end
+    
+    def initializer(path)
+      super
+    end
+  end
+end
+
+Application.initializer(ROOT_PATH)

data/bench/benchapp/config.ru

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+# Try to load vendor-ed rhosync, otherwise load the gem
+begin
+  require 'vendor/rhosync/lib/rhosync'
+rescue LoadError
+  require 'rhosync/server'
+  require 'rhosync/console/server'
+end
+
+# By default, turn on the resque web console
+require 'resque/server'
+
+ROOT_PATH = File.expand_path(File.dirname(__FILE__))
+
+# Rhosync server flags
+Rhosync::Server.disable :run
+Rhosync::Server.disable :clean_trace
+Rhosync::Server.enable  :raise_errors
+Rhosync::Server.set     :environment, :development
+Rhosync::Server.set     :secret,      '<changeme>'
+Rhosync::Server.set     :root,        ROOT_PATH
+Rhosync::Server.use     Rack::Static, :urls => ["/data"], :root => Rhosync::Server.root
+
+# Load our rhosync application
+require 'application'
+
+# Setup the url map
+run Rack::URLMap.new \
+	"/"         => Rhosync::Server.new,
+	"/resque"   => Resque::Server.new, # If you don't want resque frontend, disable it here
+	"/console"  => RhosyncConsole::Server.new # If you don't want rhosync frontend, disable it here

data/bench/benchapp/settings/license.key

@@ -0,0 +1 @@
+858fc60fadfde40273d0ac505906969318aa4931d1a2c4aeb24d98393c74379f60e226651601969874257f7f1fbda9b099ecd551a641519aa46819947fda0191

data/bench/benchapp/settings/settings.yml

@@ -0,0 +1,18 @@
+:sources:
+  MockAdapter:
+    poll_interval: 300
+  QueueMockAdapter:
+    queue: abc
+
+:development:
+  :licensefile: settings/license.key
+  :redis: localhost:6379
+  :syncserver: http://localhost:9292/application/
+:test: 
+  :licensefile: settings/license.key
+  :redis: localhost:6379
+  :syncserver: http://localhost:9292/application/
+:production: 
+  :licensefile: settings/license.key
+  :redis: localhost:6379
+  :syncserver: http://localhost:9292/application/    

data/bench/benchapp/sources/mock_adapter.rb

@@ -0,0 +1,55 @@
+class MockAdapter < SourceAdapter
+  def initialize(source,credential)
+    super(source,credential)
+  end
+ 
+  def login
+    true
+  end
+ 
+  def query(params=nil)
+    Store.lock(lock_name,1) do
+      @result = Store.get_data(db_name)
+    end
+    @result
+  end
+  
+  def create(name_value_list,blob=nil)
+    id = name_value_list['mock_id']
+    Store.lock(lock_name,1) do
+      Store.put_data(db_name,{id=>name_value_list},true) if id
+    end
+    id
+  end
+ 
+  def update(name_value_list)
+    id = name_value_list.delete('id')
+    return unless id
+    Store.lock(lock_name,1) do
+      data = Store.get_data(db_name)
+      return unless data and data[id]
+      name_value_list.each do |attrib,value|
+        data[id][attrib] = value
+      end
+      Store.put_data(db_name,data)
+    end
+  end
+ 
+  def delete(name_value_list)
+    id = name_value_list.delete('id')
+    Store.lock(lock_name,1) do
+      Store.delete_data(db_name,{id=>name_value_list}) if id
+    end
+  end
+ 
+  def db_name
+    "test_db_storage:#{@source.app_id}:#{@source.user_id}"
+  end
+  
+  def lock_name()
+    "#{db_name}:lock"
+  end
+  
+  private
+  
+end

data/bench/benchapp/sources/queue_mock_adapter.rb

@@ -0,0 +1,2 @@
+require 'mock_adapter'
+class QueueMockAdapter < MockAdapter; end

data/bench/benchapp/vendor/rhosync/lib/rhosync.rb

@@ -0,0 +1,7 @@
+
+# We're "vendoring" the rhosync/lib directory here so we can use the working copy of
+# rhosync.  Normally, you would require rhosync as a gem or vendor it here.
+path = File.join(File.dirname(__FILE__),'..','..','..','..','..','lib')
+$:.unshift path
+require File.join(path,'rhosync','server')
+require File.join(path,'rhosync','console','server')

data/bench/lib/bench/cli.rb

@@ -0,0 +1,16 @@
+require 'thor'
+
+module Bench
+  class Cli < Thor
+    include Logging
+    desc "start path/to/bench/script", "Start performance test"
+    def start(script,login,password='')
+      Bench.admin_login = login
+      Bench.admin_password = password
+      load(script)
+      Statistics.new(Bench.concurrency,Bench.iterations,
+        Bench.total_time,Bench.sessions).process.print_stats
+      logger.info "Bench completed..."
+    end
+  end
+end

data/bench/lib/bench/logging.rb

@@ -0,0 +1,18 @@
+module Bench
+  module Logging
+    def logger
+      init_logger if Log4r::Logger['main'].nil?
+      Log4r::Logger['main']
+    end
+    
+    def log_prefix
+      "[T:%03d|I:%03d]" % [@thread_id,@iteration]
+    end
+
+    protected
+    def init_logger
+      logger = Log4r::Logger.new('main')
+      logger.outputters = Log4r::Outputter.stdout
+    end
+  end
+end

data/bench/lib/bench/mock_client.rb

@@ -0,0 +1,41 @@
+require 'redis'
+$:.unshift File.join(File.dirname(__FILE__),'..','..','..','lib')
+require 'rhosync'
+include Rhosync
+  
+module Bench
+  class MockClient
+    include Logging
+        
+    def initialize(thread_id,iteration,client_id)
+      @thread_id,@iteration,@client_id = thread_id,iteration,client_id
+    end
+    
+    def parse(message)
+      msg = JSON.parse(message)
+      raise Exception.new("#{log_prefix} Wrong message format. Message: #{message.inspect}") if msg.size < 6
+      raise Exception.new("#{log_prefix} Wrong protocol version. Message: #{message.inspect}") if msg[0]['version'] != 3
+      msg.each do |p|
+        insert(p['insert']) if p['insert']
+        delete(p['delete']) if p['delete']
+      end
+    end
+    
+    def insert(objects)
+      Store.put_data(doc_type,objects,true)
+    end
+    
+    def delete(objects)
+      Store.delete_data(doc_type,objects)
+    end
+  
+    def verify(objects)
+      Store.get_data(doc_type) == objects
+    end
+      
+    def doc_type
+      "#{@client_id.to_s}:mock:cd"
+    end  
+
+  end
+end

data/bench/lib/bench/result.rb

@@ -0,0 +1,50 @@
+require 'set'
+
+module Bench
+  class Result
+    attr_accessor :last_response,:time,:marker,:url,:verb,:error,:verification_error
+    include Logging
+    include Utils
+    
+    def initialize(marker,verb,url,thread_id,iteration)
+      @marker,@verb,@url,@thread_id,@iteration = marker,verb,url,thread_id,iteration
+      @verification_error = 0
+      @time = 0
+    end
+    
+    def code
+      @last_response.code
+    end
+    
+    def body
+      @last_response.to_s
+    end
+    
+    def cookies
+      @last_response.cookies
+    end
+    
+    def headers
+      @last_response.headers
+    end
+    
+    def verify_body(expected)
+      expected,actual = JSON.parse(expected),JSON.parse(@last_response.to_s)    
+      @verification_error += compare_and_log(expected,actual,caller(1)[0].to_s)
+    end
+    
+    def verify_code(expected)
+      if expected != @last_response.code
+        logger.error "#{log_prefix} Verify error at: " + caller(1)[0].to_s
+        logger.error "#{log_prefix} Code diff: "
+        logger.error "#{log_prefix} expected: #{expected.inspect}"
+        logger.error "#{log_prefix} but got:  #{@last_response.code}"
+        @verification_error += 1
+      end
+    end
+    
+    def verify_headers
+      
+    end
+  end
+end

data/bench/lib/bench/runner.rb

@@ -0,0 +1,44 @@
+module Bench
+  class Runner
+    include Logging
+    include Timer
+    attr_reader :threads
+    
+    def initialize
+      @threads = []
+      @sessions = []
+    end
+    
+    def test(concurrency,iterations,&block)
+      thread_id = 0
+      total_time = time do
+        concurrency.times do
+          sleep rand(2)
+          thread = Thread.new(block) do |t|
+            tid, iteration = thread_id,0
+            iterations.times do
+              s = Session.new(tid,iteration)
+              @sessions << s
+              begin
+                yield Bench,s
+              rescue Exception => e
+                puts "error running script: #{e.inspect}"
+              end    
+              iteration += 1
+            end
+          end
+          thread_id += 1    
+          threads << thread
+        end
+        begin 
+          threads.each { |t| t.join }
+        rescue RestClient::RequestTimeout => e
+          logger.info "Request timed out #{e}"
+        end
+      end
+      Bench.sessions = @sessions
+      Bench.total_time = total_time
+    end    
+  end
+end  
+  

data/bench/lib/bench/session.rb

@@ -0,0 +1,65 @@
+module Bench
+  class Session
+    include Logging
+    include Timer
+    attr_accessor :cookies, :last_result, :results, :thread_id, :iteration, :client_id
+    
+    def initialize(thread_id,iteration)
+      @cookies = {}
+      @results = {}
+      @thread_id,@iteration = thread_id,iteration
+    end
+      
+    def post(marker,url,headers={})
+      @body = yield
+      _request(marker,:_post,url,headers)
+    end
+    
+    def get(marker,url,headers={})
+      params = yield if block_given?
+      url_params = url.clone
+      url_params << "?" + _url_params(params) if params
+      _request(marker,:_get,url_params,headers)          
+    end
+    
+    protected
+    def _request(marker,verb,url,headers)
+      result = Result.new(marker,verb,url,@thread_id,@iteration)
+      @results[result.marker] ||= []
+      @results[result.marker] << result
+      begin
+        result.time = time do
+          headers.merge!(:cookies => @cookies)
+            result.last_response = send(verb,url,headers)
+            @last_result = result  
+        end
+        logger.info "#{log_prefix} #{verb.to_s.upcase.gsub(/_/,'')} #{url} #{@last_result.code} #{result.time}"      
+      rescue RestClient::Exception => e
+        result.error = e
+        logger.info "#{log_prefix} #{verb.to_s.upcase.gsub(/_/,'')} #{url}"      
+        logger.error "#{log_prefix} #{e.http_code.to_s} #{e.message}\n"
+        raise e
+      end
+      @cookies = @cookies.merge(@last_result.cookies)
+      @last_result
+    end
+    
+    def _get(url,headers)
+      #logger.info "GET #{url}"
+      RestClient.get(url, headers)
+    end
+    
+    def _post(url,headers)
+      #logger.info "POST #{url}"
+      RestClient.post(url, @body, headers)
+    end
+    
+    def _url_params(params)
+      elements = []
+      params.each do |key,value|
+        elements << "#{key}=#{value}"
+      end
+      elements.join('&')
+    end
+  end
+end

data/bench/lib/bench/statistics.rb

@@ -0,0 +1,56 @@
+module Bench
+  class Statistics
+    include Logging
+    
+    def initialize(concurrency,iterations,total_time,sessions)
+      @sessions = sessions
+      @rows = {} # row key is result.marker;
+      @total_count = 0
+      @total_time = total_time
+      @concurrency,@iterations = concurrency,iterations
+    end
+    
+    def process
+      @sessions.each do |session|
+        session.results.each do |marker,results|
+          results.each do |result|
+            @rows[result.marker] ||= {}
+            row = @rows[result.marker]
+            row[:min] ||= 0.0
+            row[:max] ||= 0.0
+            row[:count] ||= 0
+            row[:total_time] ||= 0.0
+            row[:errors] ||= 0
+            row[:verification_errors] ||= 0
+            row[:min] = result.time if result.time < row[:min] || row[:min] == 0
+            row[:max] = result.time if result.time > row[:max]
+            row[:count] += 1.0
+            row[:total_time] += result.time            
+            row[:errors] += 1 if result.error
+            row[:verification_errors] += result.verification_error
+            @total_count += 1
+          end
+        end
+      end
+      self
+    end
+    
+    def average(row)
+      row[:total_time] / row[:count]
+    end
+    
+    def print_stats
+      logger.info "Statistics:"
+      @rows.each do |marker,row|
+        logger.info "Request %-15s: min: %0.4f, max: %0.4f, avg: %0.4f, err: %d, verification err: %d" % [marker, row[:min], row[:max], average(row), row[:errors], row[:verification_errors]]
+      end
+      logger.info "Verify Error       : #{Bench.verify_error}"
+      logger.info "Concurrency        : #{@concurrency}"
+      logger.info "Iterations         : #{@iterations}"
+      logger.info "Total Count        : #{@total_count}"
+      logger.info "Total Time         : #{@total_time}"
+      logger.info "Throughput(req/s)  : #{@total_count / @total_time}"
+      logger.info "Throughput(req/min): #{(@total_count / @total_time) * 60.0}"
+    end
+  end
+end

data/bench/lib/bench/test_data.rb

@@ -0,0 +1,55 @@
+require 'faker'
+require 'uuidtools'
+
+module Bench
+  module TestData
+    def get_test_data(num=1000,generate=false)
+      file = File.join(File.dirname(__FILE__),'..',"testdata","#{num}-data.txt")
+      data = nil
+      if File.exists?(file) and not generate
+        data = open(file, 'r') {|f| Marshal.load(f)}
+      else
+        data = generate_fake_data(num)
+        f = File.new(file, 'w')
+        f.write Marshal.dump(data)
+        f.close
+      end
+      data
+    end
+
+    private
+
+    PREFIX = ["Account", "Administrative", "Advertising", "Assistant", "Banking", "Business Systems", 
+      "Computer", "Distribution", "IT", "Electronics", "Environmental", "Financial", "General", "Head", 
+      "Laboratory", "Maintenance", "Medical", "Production", "Quality Assurance", "Software", "Technical", 
+      "Chief", "Senior"] unless defined? PREFIX
+    SUFFIX = ["Clerk", "Analyst", "Manager", "Supervisor", "Plant Manager", "Mechanic", "Technician", "Engineer", 
+      "Director", "Superintendent", "Specialist", "Technologist", "Estimator", "Scientist", "Foreman", "Nurse", 
+      "Worker", "Helper", "Intern", "Sales", "Mechanic", "Planner", "Recruiter", "Officer", "Superintendent",
+      "Vice President", "Buyer", "Production Supervisor", "Chef", "Accountant", "Executive"] unless defined? SUFFIX
+
+    def title
+      prefix = PREFIX[rand(PREFIX.length)]
+      suffix = SUFFIX[rand(SUFFIX.length)]
+
+      "#{prefix} #{suffix}"
+    end
+
+    def generate_fake_data(num=1000)
+      res = {}
+      num.times do |n|
+        mock_id = UUIDTools::UUID.random_create.to_s.gsub(/\-/,'')
+        res[mock_id] = {
+          "mock_id" => mock_id,
+          "FirstName" => Faker::Name.first_name,
+          "LastName" => Faker::Name.last_name,
+          "Email" =>  Faker::Internet.free_email,
+          "Company" => Faker::Company.name,
+          "JobTitle" => title,
+          "Phone1" => Faker::PhoneNumber.phone_number
+        }
+      end
+      res
+    end
+  end
+end

data/bench/lib/bench/timer.rb

@@ -0,0 +1,10 @@
+module Bench
+  module Timer
+    def time
+      start = Time.now
+      yield
+      end_time = Time.now   
+      end_time.to_f - start.to_f
+    end
+  end
+end