rhoconnect 3.0.0.rc1 → 3.0.0

data/installer/utils/unix_install_test.rb

@@ -0,0 +1,258 @@
+#! /usr/bin/env ruby
+
+require 'net/http'
+require 'constants'
+require 'rubygems'
+require 'fog'
+require 'readline'
+
+# METHODS
+
+# cmd
+# easily issue system commands in a clear way
+def cmd(cmd)
+  puts cmd
+  system(cmd)
+end #cmd
+
+# ssh_cmd
+# More easily issue commands over the ssh connection.
+def ssh_cmd(cmd)
+  puts cmd
+  puts @ssh.run(cmd)[0].stdout
+end #ssh_cmd
+
+# compile_stack_info
+# Fills in all necessary information which is stack specific
+def compile_stack_info
+  # User is stack specific but not to 
+  # be included in the fog creation hash
+  @user = @stack[:user]
+  @stack.delete :user
+
+  # This part of the package name will always be the same
+  local_file = "#{`pwd`.strip}/../../pkg/"
+
+  # Append the rest of the file name according to distribution
+  if @user == 'ubuntu'
+    @dist = { :package    => "rhoconnect_#{Constants::RC_VERSION}_all.deb",
+              :local_file => "#{local_file}rhoconnect_#{Constants::RC_VERSION}_all.deb",
+              :pkg_mgr    => 'dpkg',
+              :pkg_repo   => 'apt-get -y',
+              :deps       => Constants::DEB_DEPS }
+  elsif @user == 'root'
+    @dist = { :package    => "rhoconnect_#{Constants::RC_VERSION}_noarch.rpm",
+              :local_file => "#{local_file}rhoconnect_#{Constants::RC_VERSION}_noarch.rpm",
+              :pkg_mgr    => 'rpm',
+              :pkg_repo   => 'yes | yum',
+              :deps       => Constants::RPM_DEPS }
+  else
+    puts "Incorrect user name"
+    exit
+  end #i
+end #compile_stack_info
+
+# create_ec2_instance
+# Generates the Fog object and creates a new ec2 instance.
+def create_ec2_instance
+   make_fog
+   start_new_instance
+end #create_ec2_instance
+
+# get_access_keys
+# Retrieves the access key and secret access key from the above specified file.
+def get_access_keys
+  lines = IO.readlines Constants::ACCESS_KEY_FILE
+  @access_key = lines.first.strip
+  @secret_access_key = lines.last.strip
+end #get_access_keys
+
+# make_fog
+# Generates the Fog object used to create the new ec2 instance.
+def make_fog
+  get_access_keys
+  @fog = Fog::Compute.new(
+      :provider              => 'AWS',
+      :region                => Constants::REGION,
+      :aws_access_key_id     => @access_key,
+      :aws_secret_access_key => @secret_access_key
+  )
+end #make_fog
+
+# start_new_instance
+# Creates a new ec2 instance as per the given stack.
+def start_new_instance
+  puts "Creating new instance..."
+
+  @server = @fog.servers.create(@stack)
+  @server.wait_for { print "."; STDOUT.flush; ready? }
+  puts
+  if @server.ready?
+    # wait for all services to start on remote VM
+    puts "Waiting #{Constants::SLEEP} seconds for services to start on remote VM..."
+    Constants::SLEEP.times do
+      sleep( 1 )
+      print "."
+      STDOUT.flush
+    end #do
+    puts
+
+    # save important instance information
+    @host      = @server.dns_name
+    @server_id = @server.id
+  else
+    puts "Server timed out."
+    exit
+  end #if
+end #start_new_instance
+
+# establish_ssh_connection
+# Establishes the ssh connection needed to issue commands to the remote machine.
+def establish_ssh_connection
+  @ssh = Fog::SSH.new( @host, @user, :keys => Constants::SSH_KEY )
+end #establish_ssh_connection
+
+# establish_scp_connection
+# Establishes a gateway through which to transfer data
+def establish_scp_connection
+  @scp = Fog::SCP.new( @host, @user, :keys => Constants::SSH_KEY )
+end #create_scp_connection
+
+# transfer 
+# transfers given data to remote machine via scp
+def transfer(local_data, remote_data)
+  puts "Attempting to establish SCP connection..."
+  @scp.upload( local_data, remote_data ) do |cd, name, sent, total|
+    print "\r#{name}: #{(sent.to_f * 100 / total.to_f).to_i}%"
+  end #do
+  puts
+end #establish_scp_connection
+
+# transfer_rhoconnect
+# Transfers the rhoconnect package over SCP
+def transfer_rhoconnect
+  transfer @dist[:local_file], Constants::REMOTE_HOME
+end #transfer_rhoconnect
+
+# destroy_ec2_instance
+# Terminates the current ec2 instance
+def destroy_ec2_instance
+  @server.destroy
+  puts "Terminating Instance."
+end #destroy_ec2_instance
+
+# install_dependencies
+# Installs all dependant packages on the remove machine
+def install_dependencies
+  dep_install_string = "sudo #{@dist[:pkg_repo]} install"
+  @dist[:deps].each do |dep|
+    dep_install_string << " #{dep}"
+  end #do
+  ssh_cmd "sudo #{@dist[:pkg_repo]} update"
+  ssh_cmd dep_install_string
+  puts
+end #install_dependencies
+
+# install_package
+# Issues commands to the remote machine to start the installation
+def install_package
+  install_dependencies
+  ssh_cmd "sudo #{@dist[:pkg_mgr]} -i #{@dist[:package]}"
+  puts
+end #install_package
+
+def start_servers
+  puts "Starting Redis server"
+  ssh_cmd "sudo /etc/init.d/redis start"
+  puts "Starting nginx server"
+  ssh_cmd "sudo /etc/init.d/nginx start"
+  puts
+end # start_servers
+
+def check_rc_service
+  request = Net::HTTP.get_response(@host, '/console')
+  puts "Host: #{@host}"
+  puts "Code: #{request.code} Message: #{request.message}"
+  if request.code == '200'
+    puts "Rhoconnect service up!"
+  else
+    puts "Rhoconnect service down :("
+  end #if
+end #check_rc_service
+
+def compute_time(start_time, end_time)
+  time_delta = (end_time - start_time)
+
+  if time_delta >= 60
+    time_delta /= 60
+    time_units = 'minute'
+    if time_delta > 1
+      time_units << 's'
+    end #if
+  else
+    time_units = 'seconds'
+  end #if
+
+"Test completed in #{time_delta} #{time_units}"
+end #compute_time
+
+# SCRIPT
+
+# Create Packages
+
+# Creating debian package
+cmd "sudo rake build:deb"
+# Make sure package was created successfully
+if $? != 0
+  puts "Debian package was not created successfully" + 
+       "Exiting..."
+  exit 1
+end #if
+
+# Keep commented until running on a machine with rpm
+# Creating RPM package
+#cmd "sudo rake build:rpm"
+# Make sure package was created successfully
+#if $? != 0
+#  puts "RPM package was not created successfully" + 
+#       "Exiting..."
+#  exit 1
+#end #if
+
+# Test Package installations
+start_time = Time.now
+Constants::STACKS.each do |stack|
+  begin
+    @stack = stack 
+    compile_stack_info
+
+    create_ec2_instance
+ 
+    # Establish connections
+    establish_ssh_connection
+    establish_scp_connection
+
+    # Transfer the Rhoconnect package to the remote machine
+    transfer_rhoconnect
+ 
+    puts "Installing rhoconnect package.\n" + 
+         "This may take a while..."
+    install_package
+
+    # Start the redis and nginx servers on the remote machine
+    start_servers
+
+    # Check the status of the rhoconnect service
+    check_rc_service
+  rescue => e
+    puts "OH NOEZ, Exception!!"
+    puts e.inspect
+    puts e.backtrace
+  ensure
+    if @server != nil
+      destroy_ec2_instance
+    end #if
+  end #begin
+end #do
+end_time = Time.now
+puts compute_time start_time end_time

data/lib/rhoconnect/api/source/push_deletes.rb

@@ -3,6 +3,9 @@ Server.api :push_deletes, :source do |params,user|
   # if source does not exist create one for dynamic adapter
   source = Source.create({:name => params[:source_id]},{:app_id => APP_NAME,:user_id=>params[:user_id]}) unless source
   source_sync = SourceSync.new(source)
-  source_sync.push_deletes(params[:objects])
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  rebuild_md = params[:rebuild_md].nil? ? true : params[:rebuild_md]
+  source_sync.push_deletes(params[:objects],timeout,raise_on_expire,rebuild_md)
   'done'
 end

data/lib/rhoconnect/api/source/push_objects.rb

@@ -3,6 +3,9 @@ Server.api :push_objects, :source do |params,user|
   # if source does not exist create one for dynamic adapter
   source = Source.create({:name => params[:source_id]},{:app_id => APP_NAME,:user_id=>params[:user_id]}) unless source
   source_sync = SourceSync.new(source)
-  source_sync.push_objects(params[:objects])
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  rebuild_md = params[:rebuild_md].nil? ? true : params[:rebuild_md]
+  source_sync.push_objects(params[:objects],timeout,raise_on_expire,rebuild_md)
   'done'
 end

data/lib/rhoconnect/document.rb

@@ -13,6 +13,14 @@ module Document
     Store.put_data(docname(doctype),data,append)
   end
   
+  def update_objects(doctype,updates)
+    Store.update_objects(docname(doctype),updates)
+  end
+  
+  def remove_objects(doctype,deletes)
+    Store.delete_objects(docname(doctype),deletes)
+  end
+  
   def put_value(doctype,data)
     Store.put_value(docname(doctype),data)
   end

data/lib/rhoconnect/source.rb

@@ -262,42 +262,16 @@ module Rhoconnect
       self.poll_interval == 0 or 
       (self.poll_interval != -1 and self.read_state.refresh_time <= Time.now.to_i)
     end
-        
-    def if_need_refresh(client_id=nil,params=nil)
-      need_refresh = check_refresh_time
-      yield client_id,params if need_refresh
-    end
     
-    def update_refresh_time(query_failure = false)
-      if self.poll_interval == 0
-        self.read_state.refresh_time = Time.now.to_i + self.poll_interval
-        return
-      end
-      
-      allowed_update = true
-      # reset number of retries on succesfull query
-      # or if last refresh was more than 'poll_interval' time ago
-      if not query_failure or (Time.now.to_i - self.read_state.refresh_time >= self.poll_interval)
-        self.read_state.retry_counter = 0 
-      end
-      
-      # do not reset the refresh time on failure
-      # if retry limit is not reached
-      if query_failure
-        if self.read_state.retry_counter < self.retry_limit
-          allowed_update = false
-          self.read_state.increment!(:retry_counter)
-        # we have reached the limit - update the refresh time
-        # and reset the counter
-        else
-          self.read_state.retry_counter = 0 
-        end
-      end
-      if allowed_update
-        self.read_state.refresh_time = Time.now.to_i + self.poll_interval
+    def if_need_refresh(client_id=nil,params=nil)
+      need_refresh = lock(:md) do |s|
+        check = check_refresh_time
+        s.read_state.refresh_time = Time.now.to_i + s.poll_interval if check
+        check
       end
+      yield client_id,params if need_refresh
     end
-    
+        
     def is_pass_through?
       self.pass_through and self.pass_through == 'true'
     end

data/lib/rhoconnect/source_adapter.rb

@@ -55,16 +55,20 @@ module Rhoconnect
     
     def sync
       if @result and @result.empty?
-        @source.flash_data(:md)
-        @source.put_value(:md_size,0)
+        @source.lock(:md) do |s|
+          s.flash_data(:md)
+          s.put_value(:md_size,0)
+        end
       else
         if @result
           Store.put_data(@tmp_docname,@result) 
           @stash_size += @result.size
         end  
-        @source.flash_data(:md)
-        Store.rename(@tmp_docname,@source.docname(:md))
-        @source.put_value(:md_size,@stash_size)
+        @source.lock(:md) do |s|
+          s.flash_data(:md)
+          Store.rename(@tmp_docname,s.docname(:md))
+          s.put_value(:md_size,@stash_size)
+        end
       end
     end
     

data/lib/rhoconnect/source_sync.rb

@@ -90,17 +90,11 @@ module Rhoconnect
     
     def do_query(params=nil)
       result = nil
-      @source.lock(:md) do
-        @source.if_need_refresh do
-          Stats::Record.update("source:query:#{@source.name}") do
-            if _auth_op('login')
-              result = self.read(nil,params)
-              _auth_op('logoff')
-            end
-            # update refresh time
-            query_failure = Store.get_keys(@source.docname(:errors)).size > 0
-            @source.update_refresh_time(query_failure)
-          end
+      @source.if_need_refresh do
+        Stats::Record.update("source:query:#{@source.name}") do
+          return if _auth_op('login') == false
+          result = self.read(nil,params)
+          _auth_op('logoff')
         end  
       end
       result
@@ -113,31 +107,57 @@ module Rhoconnect
         @source.app_id,@source.user_id,client_id,params)
     end
     
-    def push_objects(objects,timeout=10,raise_on_expire=false)
+    def push_objects(objects,timeout=10,raise_on_expire=false,rebuild_md=true)
       @source.lock(:md,timeout,raise_on_expire) do |s|
-        doc = @source.get_data(:md)
-        orig_doc_size = doc.size
-        objects.each do |id,obj|
-          doc[id] ||= {}
-          doc[id].merge!(obj)
-        end  
-        diff_count = doc.size - orig_doc_size
-        @source.put_data(:md,doc)
+        diff_count = 0
+        # in case of rebuild_md
+        # we clean-up and rebuild the whole :md doc
+        # on every request
+        if(rebuild_md)
+          doc = @source.get_data(:md)
+          orig_doc_size = doc.size
+          objects.each do |id,obj|
+            doc[id] ||= {}
+            doc[id].merge!(obj)
+          end  
+          diff_count = doc.size - orig_doc_size
+          @source.put_data(:md,doc)
+        else
+          # if rebuild_md == false
+          # we only operate on specific set values
+          # which brings a big optimization
+          # in case of small transactions
+          diff_count = @source.update_objects(:md, objects)
+        end
+        
         @source.update_count(:md_size,diff_count)
       end      
     end    
 
-    def push_deletes(objects,timeout=10,raise_on_expire=false)
+    def push_deletes(objects,timeout=10,raise_on_expire=false,rebuild_md=true)
       @source.lock(:md,timeout,raise_on_expire) do |s|
-        doc = @source.get_data(:md)
-        orig_doc_size = doc.size
-        objects.each do |id|
-          doc.delete(id)
-        end  
-        diff_count = doc.size - orig_doc_size
-        @source.put_data(:md,doc)
+        diff_count = 0
+        if(rebuild_md)
+          # in case of rebuild_md
+          # we clean-up and rebuild the whole :md doc
+          # on every request
+          doc = @source.get_data(:md)
+          orig_doc_size = doc.size
+          objects.each do |id|
+            doc.delete(id)
+          end  
+          diff_count = doc.size - orig_doc_size
+          @source.put_data(:md,doc)
+        else
+          # if rebuild_md == false
+          # we only operate on specific set values
+          # which brings a big optimization
+          # in case of small transactions
+          diff_count = -@source.remove_objects(:md, objects)
+        end
+        
         @source.update_count(:md_size,diff_count)
-      end      
+      end    
     end
     
     private

data/lib/rhoconnect/store.rb

@@ -47,6 +47,57 @@ module Rhoconnect
         end
         true
       end
+      
+      # updates objects for a given doctype, source, user
+      # create new objects if necessary
+      def update_objects(dockey, data={})
+        return 0 unless dockey and data
+        
+        new_object_count = 0
+        doc = get_data(dockey)
+        @@db.pipelined do
+          data.each do |key,value|
+            is_create = doc[key].nil?
+            new_object_count += 1 if is_create
+            value.each do |attrib,value|
+              next if _is_reserved?(attrib, value)
+              
+              new_element = setelement(key,attrib,value)
+              element_exists = is_create ? false : doc[key].has_key?(attrib)
+              if element_exists
+                existing_element = setelement(key,attrib,doc[key][attrib])
+                if existing_element != new_element
+                  @@db.srem(dockey, existing_element)
+                  @@db.sadd(dockey, new_element)
+                end
+              else
+                @@db.sadd(dockey, new_element)
+              end
+            end
+          end
+        end
+        new_object_count
+      end
+      
+      # Removes objects from a given doctype,source,user
+      def delete_objects(dockey,data=[])
+        return 0 unless dockey and data
+        
+        deleted_object_count = 0
+        doc = get_data(dockey)
+        @@db.pipelined do
+          data.each do |id|
+            if doc[id]
+              doc[id].each do |name,value|
+                @@db.srem(dockey, setelement(id,name,value))
+              end
+              deleted_object_count += 1
+            end
+            doc.delete(id)
+          end
+        end
+        deleted_object_count
+      end
     
       # Adds a simple key/value pair
       def put_value(dockey,value)

data/lib/rhoconnect/tasks.rb

@@ -270,7 +270,7 @@ namespace :rhoconnect do
     end
   end
   
-  desc "Start rhoconnect server in bg mode (Rhostudio)"
+#  desc "Start rhoconnect server in bg mode (Rhostudio) - this is an internal command"
   task :startbg do
     cmd = (jruby?) ? trinidad? : (thin? || mongrel? || report_missing_server)
     Thread.new {  
@@ -288,6 +288,25 @@ namespace :rhoconnect do
     exit
   end
   
+#  desc "Start rhoconnect server in debug mode (Rhostudio) - this is an internal command"
+  task :startdebug do
+    cmd = (jruby?) ? trinidad? : (thin? || mongrel? || report_missing_server)
+    ENV['DEBUG'] = 'yes'
+    Thread.new {  
+      if windows?
+        puts 'Starting rhoconnect...'
+        system("#{cmd} config.ru -P #{rhoconnect_pid}")
+      elsif jruby?
+        puts 'Starting rhoconnect in jruby environment...'
+        system("#{cmd}")
+      else
+        system("#{cmd} config.ru -P #{rhoconnect_pid}")
+      end
+    }
+  
+    exit
+  end
+  
   desc "run rhoconnect console"
   task :console, :environment do |t, args|
     if RedisRunner.running?        

data/lib/rhoconnect/version.rb

@@ -1,3 +1,3 @@
 module Rhoconnect
-  VERSION = '3.0.0.rc1'
+  VERSION = '3.0.0'
 end

data/rhoconnect.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
   s.rubyforge_project = nil
   
   s.files        = %w( 
-    CHANGELOG CREDITS Gemfile Gemfile.lock install.sh README.md Rakefile LICENSE Rakefile rhoconnect.gemspec 
+    CHANGELOG.md CREDITS Gemfile Gemfile.lock install.sh README.md Rakefile LICENSE Rakefile rhoconnect.gemspec 
   )
   s.files       += Dir.glob("bench/**/*")
   s.files       += Dir.glob("doc/**/*")

data/spec/source_sync_spec.rb

@@ -231,55 +231,6 @@ describe "SourceSync" do
           verify_read_operation_with_error('query')
         end
         
-        it "should do query with exception raised and update refresh time only after retries limit is exceeded" do
-          @s.retry_limit = 1
-          msg = "Error during query"
-          @ss.should_receive(:log).with("SourceAdapter raised query exception: #{msg}")
-          @ss.should_receive(:log).with(anything)
-          set_test_data('test_db_storage',{},msg,"query error")
-          res = @ss.do_query
-          verify_result(@s.docname(:md) => {},
-            @s.docname(:errors) => {'query-error'=>{'message'=>msg}})
-          # 1) if retry_limit is set to N - then, first N retries should not update refresh_time
-          @s.read_state.retry_counter.should == 1
-          @s.read_state.refresh_time.should <= Time.now.to_i
-          
-          # try once more and fail again
-          @ss.should_receive(:log).with("SourceAdapter raised query exception: #{msg}")
-          @ss.should_receive(:log).with(anything)
-          set_test_data('test_db_storage',{},msg,"query error")
-          res = @ss.do_query
-          verify_result(@s.docname(:md) => {},
-            @s.docname(:errors) => {'query-error'=>{'message'=>msg}})
-          
-          # 2) if retry_limit is set to N and number of retries exceeded it - update refresh_time
-          @s.read_state.retry_counter.should == 0
-          @s.read_state.refresh_time.should > Time.now.to_i
-        end
-        
-        it "should do query with exception raised and restore state with succesfull retry" do
-          @s.retry_limit = 1
-          msg = "Error during query"
-          @ss.should_receive(:log).with("SourceAdapter raised query exception: #{msg}")
-          @ss.should_receive(:log).with(anything)
-          set_test_data('test_db_storage',{},msg,"query error")
-          res = @ss.do_query
-          verify_result(@s.docname(:md) => {},
-            @s.docname(:errors) => {'query-error'=>{'message'=>msg}})
-          # 1) if retry_limit is set to N - then, first N retries should not update refresh_time
-          @s.read_state.retry_counter.should == 1
-          @s.read_state.refresh_time.should <= Time.now.to_i
-          
-          # try once more (with success)
-          expected = {'1'=>@product1,'2'=>@product2}
-          set_test_data('test_db_storage',expected)
-          @ss.do_query 
-          verify_result(@s.docname(:md) => expected, 
-            @s.docname(:errors) => {})
-          @s.read_state.retry_counter.should == 0
-          @s.read_state.refresh_time.should > Time.now.to_i
-        end
-        
         it "should do query with exception raised and update refresh time if retry_limit is 0" do
           @s.retry_limit = 0
           msg = "Error during query"
@@ -289,7 +240,6 @@ describe "SourceSync" do
           res = @ss.do_query
           verify_result(@s.docname(:md) => {},
             @s.docname(:errors) => {'query-error'=>{'message'=>msg}})
-          #  if poll_interval is set to 0 - refresh time should be updated
           @s.read_state.retry_counter.should == 0
           @s.read_state.refresh_time.should > Time.now.to_i
         end