rhoconnect 3.1.2 → 3.2.0.beta1

data/installer/utils/nix_install_test.rb

@@ -37,9 +37,9 @@ def build_type
   when 'beta'
     'beta-packages'
   when 'release'
-    'release-packages'
+    'packages'
   else
-    'nightly-packages'
+    'packages'
   end #case
 end #build_type
 
@@ -60,28 +60,28 @@ def compile_stack_info
   # Append the rest of the file name according to distribution
   if @user == 'ubuntu'
 
-    @dist = { :flavor        => "ubuntu",
-              :package       => "rhoconnect_#{Constants::RC_VERSION}_all.deb",
-              :local_file    => "#{local_file}rhoconnect_#{Constants::RC_VERSION}_all.deb",
-              :pkg_mgr       => 'dpkg',
-              :pkg_type      => 'DEB',
-              :pkg_repo      => 'apt-get',
-              :deps          => Constants::DEB_DEPS,
+    @dist = { :flavor => "ubuntu",
+              :package => "rhoconnect_#{Constants::RC_VERSION}_all.deb",
+              :local_file => "#{local_file}rhoconnect_#{Constants::RC_VERSION}_all.deb",
+              :pkg_mgr => 'dpkg',
+              :pkg_type => 'DEB',
+              :pkg_repo => 'apt-get',
+              :deps => Constants::DEB_DEPS,
               :repo_src_file => '/etc/apt/sources.list',
-              :repo_str      => '\n' + 
-                                '# This is the repository for rhoconnect packages\n' + 
+              :repo_str => '\n' +
+                                '# This is the repository for rhoconnect packages\n' +
                                 "deb http://#{BUCKET}.s3.amazonaws.com/#{channel}/deb rhoconnect main" }
   elsif @user == 'root'
-    @dist = { :flavor        => "centos",
-              :package       => "rhoconnect-#{Constants::RC_VERSION}.noarch.rpm",
-              :local_file    => "#{local_file}rhoconnect-#{Constants::RC_VERSION}.noarch.rpm",
-              :pkg_mgr       => 'rpm',
-              :pkg_type      => 'RPM',
-              :pkg_repo      => 'yum',
-              :deps          => Constants::RPM_DEPS,
+    @dist = { :flavor => "centos",
+              :package => "rhoconnect-#{Constants::RC_VERSION}.noarch.rpm",
+              :local_file => "#{local_file}rhoconnect-#{Constants::RC_VERSION}.noarch.rpm",
+              :pkg_mgr => 'rpm',
+              :pkg_type => 'RPM',
+              :pkg_repo => 'yum',
+              :deps => Constants::RPM_DEPS,
               :repo_src_file => '/etc/yum.repos.d/rhoconnect.repo',
-              :repo_str      => '[rhoconnect]\n' + 
-                                'name=Rhoconnect\n' + 
+              :repo_str => '[rhoconnect]\n' +
+                                'name=Rhoconnect\n' +
                                 "baseurl=http://#{BUCKET}.s3.amazonaws.com/#{channel}/rpm" +
                                 '\nenabled=1\n' +
                                 'gpgcheck=0\n' }
@@ -112,9 +112,9 @@ end #get_access_keys
 def make_fog
   get_access_keys
   @fog = Fog::Compute.new(
-      :provider              => 'AWS',
-      :region                => Constants::REGION,
-      :aws_access_key_id     => @access_key,
+      :provider => 'AWS',
+      :region => Constants::REGION,
+      :aws_access_key_id => @access_key,
       :aws_secret_access_key => @secret_access_key
   )
 end #make_fog
@@ -138,7 +138,7 @@ def start_new_instance
     puts
 
     # save important instance information
-    @host      = @server.dns_name
+    @host = @server.dns_name
     @server_id = @server.id
   else
     puts "Server timed out."
@@ -162,11 +162,11 @@ end #destroy_ec2_instance
 # prepare_sources
 # Prepares the remote machine's repo source list to be able to download rhoconnect
 def prepare_sources
-  puts "preparing sources..."  
+  puts "preparing sources..."
 
   ssh_cmd "sudo touch #{@dist[:repo_src_file]}" if @dist[:flavor] == 'centos'
   filename = @dist[:repo_src_file]
-  src_str  = @dist[:repo_str]
+  src_str = @dist[:repo_str]
 
   # Create file for yum rhoconnect sources
   # Get current permissions of file
@@ -186,7 +186,7 @@ end #prepare_sources
 # Issues commands to the remote machine to start the installation
 def install_package
   prepare_sources
-  puts "Installing rhoconnect package.\n" + 
+  puts "Installing rhoconnect package.\n" +
        "This may take a while...\n\n"
   ssh_cmd "yes | sudo #{@dist[:pkg_repo]} install rhoconnect"
 end #install_package
@@ -197,7 +197,7 @@ def start_servers
   max_attempts = 100
   puts
   
-  ['Redis', 'Nginx'].each do |program|
+  ['Redis', 'Nginx', 'thin'].each do |program|
     attempts = 0
     while @ssh.run("pgrep #{program.downcase}")[0].stdout.strip == "" and
           attempts < max_attempts
@@ -221,14 +221,15 @@ def check_rc_service
   puts "Host: #{@host}"
 
   puts "Checking RC service."
-  request = nil  #Defined outside block as to not limit the scope of request to the do block.
+  request = nil #Defined outside block as to not limit the scope of request to the do block.
   2.times do
     request = Net::HTTP.get_response(@host, '/')
   end #do
 
   puts "Code: #{request.code} Message: #{request.message}"
 
-  if request.code == '200'
+  if request.code == '200' or
+     request.code == '302'
     puts "Rhoconnect service up!"
   else
     puts "Failed to connect to rhoconnect service."
@@ -261,7 +262,7 @@ Constants::STACKS.each do |stack|
   start_time = Time.now
 
   begin
-    @stack = stack 
+    @stack = stack
     compile_stack_info
 
     puts "\n" +

data/installer/utils/nix_installation.rake

@@ -4,4 +4,4 @@ namespace :rc_pkg do
     # Run installation test
     ruby "installer/utils/nix_install_test.rb"
   end #:test
-end #:rc_pkg
+end #:rc_pkg

data/installer/utils/package_upload/repos.rake

@@ -18,18 +18,20 @@ task "build:repos", :build_type, :build_number do |t, args|
 
   # CONSTANTS
 
-  PKG_DIR     = case build_type
-                when 'release'
-                  '/packages'
-                when 'test'
-                  '/test-packages'
-                when 'beta'
-                  '/beta-packages'
-                else 'nightly'
-                  '/nightly-packages'
-                end #case
-                   
-  BUCKET      = 'rhoconnect'
+  RHOCONNECT_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', '..'))
+
+  PKG_DIR         = case build_type
+                    when 'release'
+                      '/packages'
+                    when 'test'
+                      '/test-packages'
+                    when 'beta'
+                      '/beta-packages'
+                    else 'nightly'
+                      '/nightly-packages'
+                    end #case
+
+  BUCKET          = 'rhoconnect'
 
   def cmd(cmd)
     puts cmd
@@ -70,6 +72,8 @@ task "build:repos", :build_type, :build_number do |t, args|
   end #prepare_destination
 
   def copy_files
+    # Move back into rhoconnect repo root first
+    Dir.chdir RHOCONNECT_ROOT
     # Copy the packages to their respective directory
     Find.find('./pkg') do |file|
       if !FileTest.directory?(file)
@@ -88,25 +92,25 @@ task "build:repos", :build_type, :build_number do |t, args|
   end #copy_files
 
   prepare_destination
-	
+  
   copy_files
   
-  if build_type != "release" &&
-     build_type != "test"
-    # Change name of packages to include build number
-    ['deb', 'rpm'].each do |arch|
-      Find.find("#{PKG_DIR}") do |file|
-        if !File.directory?(file) &&
-           file =~ /#{arch}$/
-          file_to_rename = File.open(file, 'r')
-          old_name = File.expand_path(file)
-          puts "Old Name:\t#{old_name}"
-          new_name = old_name.gsub(/(#{arch})$/, "build-#{build_number.to_s}.\\1" )
-          File.rename(old_name, new_name)
-        end #if
-      end #do
-    end #do
-  end #if
+ # if build_type != "release" &&
+ #    build_type != "test"
+ #   # Change name of packages to include build number
+ #   ['deb', 'rpm'].each do |arch|
+ #     Find.find("#{PKG_DIR}") do |file|
+ #       if !File.directory?(file) &&
+ #          file =~ /#{arch}$/
+ #         file_to_rename = File.open(file, 'r')
+ #         old_name = File.expand_path(file)
+ #         puts "Old Name:\t#{old_name}"
+ #         new_name = old_name.gsub(/(#{arch})$/, "build-#{build_number.to_s}.\\1" )
+ #         File.rename(old_name, new_name)
+ #       end #if
+ #     end #do
+ #   end #do
+ # end #if
 
   # REPOIFY!
   cmd "sudo reprepro -b #{PKG_DIR}/deb includedeb rhoconnect #{PKG_DIR}/deb/#{@deb_pkg}"
@@ -121,4 +125,4 @@ task "build:repos", :build_type, :build_number do |t, args|
     cmd "sudo ruby ./installer/utils/package_upload/s3_upload.rb #{PKG_DIR}/#{dir} #{BUCKET}" 
   end #do
   cmd "sudo ruby ./installer/utils/package_upload/s3_upload.rb #{PKG_DIR}/SHA1 #{BUCKET}"
-end #build:repos
+end #build:repos

data/installer/utils/package_upload/s3_upload.rb

@@ -7,7 +7,7 @@ require 'find'
 # CONSTANTS
 
 START_DIR = ARGV[0]
-BUCKET    = ARGV[1]
+BUCKET = ARGV[1]
 USER_DATA = `wget -q -O - http://169.254.169.254/latest/user-data`.split("\n")
 
 # METHODS
@@ -42,7 +42,7 @@ end #check_params
 
 # Retrieves S3 credential information
 def get_keys
-   access_key        = USER_DATA[0].to_s.strip
+   access_key = USER_DATA[0].to_s.strip
    secret_access_key = USER_DATA[1].to_s.strip
    keys = { :access_key_id => access_key,
             :secret_access_key => secret_access_key}
@@ -66,7 +66,7 @@ def upload(file)
 end #upload
 
 # Traverses list of file to upload and calls Upload to upload them
-def upload_files 
+def upload_files
   # Get list of files do be added
   number_of_files = 0
   paths = []

data/lib/rhoconnect/api/source/fast_delete.rb

@@ -0,0 +1,10 @@
+Server.api :fast_delete, :source do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  # if source does not exist create one for dynamic adapter
+  source = Source.create({:name => params[:source_id]},{:app_id => APP_NAME,:user_id=>params[:user_id]}) unless source
+  source_sync = SourceSync.new(source)
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  source_sync.fast_delete(params[:data], timeout,raise_on_expire)
+  'done'
+end

data/lib/rhoconnect/api/source/fast_insert.rb

@@ -0,0 +1,10 @@
+Server.api :fast_insert, :source do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  # if source does not exist create one for dynamic adapter
+  source = Source.create({:name => params[:source_id]},{:app_id => APP_NAME,:user_id=>params[:user_id]}) unless source
+  source_sync = SourceSync.new(source)
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  source_sync.fast_insert(params[:data], timeout,raise_on_expire)
+  'done'
+end

data/lib/rhoconnect/api/source/fast_update.rb

@@ -0,0 +1,10 @@
+Server.api :fast_update, :source do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  # if source does not exist create one for dynamic adapter
+  source = Source.create({:name => params[:source_id]},{:app_id => APP_NAME,:user_id=>params[:user_id]}) unless source
+  source_sync = SourceSync.new(source)
+  timeout = params[:timeout] || 10
+  raise_on_expire = params[:raise_on_expire] || false
+  source_sync.fast_update(params[:delete_data], params[:data],timeout,raise_on_expire)
+  'done'
+end

data/lib/rhoconnect/api/source/get_source_params.rb

@@ -1,3 +1,3 @@
 Server.api :get_source_params, :source do |params,user|
-  Source.load(params[:source_id],{:app_id => APP_NAME,:user_id => '*'}).to_array.to_json
+  Source.load(params[:source_id],{:app_id => APP_NAME,:user_id => "*"}).to_array.to_json
 end

data/lib/rhoconnect/api/source/list_sources.rb

@@ -1,12 +1,12 @@
 Server.api :list_sources, :source  do |params,user|
   sources = App.load(APP_NAME).sources
-  if params[:partition_type].nil? or params[:partition_type] == 'all'
+  if params['partition_type'].nil? or params['partition_type'] == 'all'
     sources.to_json 
   else
     res = []
     sources.each do |name|
       s = Source.load(name,{:app_id => APP_NAME,:user_id => '*'})
-      if s.partition_type and s.partition_type == params[:partition_type].to_sym
+      if s and s.partition_type and s.partition_type == params['partition_type'].to_sym
         res << name 
       end
     end  

data/lib/rhoconnect/api/source/save_adapter.rb

@@ -1,3 +1,3 @@
 Server.api :save_adapter, :source do |params,user|
-  Rhoconnect.appserver = params[:attributes]['adapter_url']
+  Rhoconnect.appserver = params['adapter_url']
 end

data/lib/rhoconnect/api/source/update_source_params.rb

@@ -0,0 +1,6 @@
+Server.api :update_source_params, :source do |params,user|
+  source = Source.load(params[:source_name],
+    {:app_id => APP_NAME, :user_id => params[:user_name]})
+  source.update_fields(params[:data])
+  ''
+end

data/lib/rhoconnect/async.rb

@@ -0,0 +1,82 @@
+require 'rack/fiber_pool'
+require 'async-rack'
+require 'eventmachine'
+
+module Rhoconnect
+  module Synchrony
+    def setup_sessions(builder)
+      options = {}
+      if settings.respond_to?(:fiberpool_size)
+        options[:size] = settings.fiberpool_size
+      end
+      options[:rescue_exception] = handle_exception 
+      builder.use Rack::FiberPool, options unless test?
+      super
+    end
+
+    def handle_exception
+      Proc.new do |env, e|
+        if settings.show_exceptions?
+          request = Sinatra::Request.new(env)
+          printer = Sinatra::ShowExceptions.new(proc{ raise e })
+          s, h, b = printer.call(env)
+          [s, h, b]
+        else
+          [500, {}, ""]
+        end
+      end
+    end
+  end
+
+  module AsyncHelpers
+    def self.included(klass)
+      (klass.instance_methods & self.instance_methods).each do |method|
+        klass.instance_eval{remove_method method.to_sym}
+      end
+    end
+    
+    def catch_all
+      res = nil
+      begin
+        res = catch(:halt) { yield }
+      rescue ApiException => ae
+        res = [ae.error_code, ae.message]
+      rescue Exception => e
+        log e.message + e.backtrace.join("\n")
+        res = [500, e.message]
+      end
+      res
+    end
+  
+    def execute_api_call(client_call = false)
+        f = Fiber.current
+        operation = proc {
+          if client_call or check_api_token
+            catch_all do
+              res = yield params, (client_call ? current_user : api_user), self
+              if params.has_key? :warning
+                Rhoconnect.log params[:warning]
+                response.headers['Warning'] = params[:warning]
+              end
+              res
+            end
+          else
+            [422, "No API token provided"]
+          end
+        }
+        result = nil
+        callback = proc { |proc_res| result = proc_res; f.resume }
+    
+        EventMachine.defer operation, callback
+        Fiber.yield
+        # we can not throw exceptions across threads
+        # so we analyze it in the main thread after the
+        # request has been processed and if result
+        # has error code - then we throw :halt here
+        if Array === result and Fixnum === result.first
+          throw :halt, result
+        end
+        result
+    end
+  end
+end

data/lib/rhoconnect/bulk_data/bulk_data.rb

@@ -54,7 +54,11 @@ module Rhoconnect
       end
       true
     end
-    
+
+    def delete_files
+      FileUtils.rm Dir.glob(File.join(Rhoconnect.base_directory, "#{self.url}*"))
+    end
+
     class << self
       def create(fields={})
         fields[:id] = fields[:name]

data/lib/rhoconnect/client_sync.rb

@@ -261,14 +261,14 @@ module Rhoconnect
         raise ArgumentError.new(UNKNOWN_CLIENT) unless client
         name = BulkData.get_name(partition,client.user_id)
         data = BulkData.load(name)
-        
+
         sources = client.app.partition_sources(partition,client.user_id)
         return {:result => :nop} if sources.length <= 0
-        
+
         do_bd_sync = data.nil?
         do_bd_sync = (data.completed? and 
             (data.refresh_time <= Time.now.to_i or !data.dbfiles_exist?)) unless do_bd_sync
-               
+
         if do_bd_sync  
           data.delete if data
           data = BulkData.create(:name => name,
@@ -281,6 +281,21 @@ module Rhoconnect
         
         if data and data.completed? and data.dbfiles_exist?
           client.update_clientdoc(sources)
+          sources.each do |src|
+            s = Source.load(src, {:user_id => client.user_id, :app_id => client.app_id})
+            errordoc = s.docname(:errors)
+            errors = {}
+            Store.lock(errordoc) do
+              errors = Store.get_data(errordoc)
+            end
+            unless errors.empty?
+              # FIXME: :result => :bulk_sync_error, :errors => "#{errors}"
+              log "Bulk sync errors are found in #{src}: #{errors}"
+              # Delete all related bulk files
+              data.delete_files
+              return {:result => :url, :url => ''}
+            end
+          end
           {:result => :url, :url => data.url}
         elsif data
           {:result => :wait}

data/lib/rhoconnect/console/rhoconnect_api.rb

@@ -29,7 +29,7 @@ module RhoconnectApi
         Server.save_adapter({:attributes => {'adapter_url' => adapter_url}},nil)
       else  
         RestClient.post("#{server}/api/source/save_adapter",
-          {:api_token => token,:attributes => {:adapter_url => adapter_url}}.to_json, 
+          {:api_token => token,:adapter_url => adapter_url}.to_json, 
             :content_type => :json)
       end
     end
@@ -37,7 +37,7 @@ module RhoconnectApi
     def create_user(server,token,login,password)
       if directcall?(server) and verify_token(token)
         Server.create_user({:attributes => {'login' => login, 'password' => password}},nil)
-      else  
+      else
         RestClient.post("#{server}/api/user/create_user",
           {:api_token => token,:attributes => {:login => login, :password => password}}.to_json, 
             :content_type => :json)

data/lib/rhoconnect/console/server.rb

@@ -1,32 +1,5 @@
-$:.unshift File.dirname(__FILE__)
-require 'rubygems'
-require 'sinatra/base'
-require 'erb'
-require 'json'
-require 'rhoconnect_api'
-
-module RhoconnectConsole  
-  class << self
-    ROOT_DIR = File.dirname(File.expand_path(__FILE__)) unless defined? ROOT_DIR
-    def root_path(*args)
-      File.join(ROOT_DIR, *args)
-    end
-  end  
-
-  class Server < Sinatra::Base
-    set :views,         RhoconnectConsole::root_path("app","views")
-    set :public_folder, RhoconnectConsole::root_path("app","public")
-    set :static,        true    
-    use Rack::Session::Cookie
-    HEROKU_NAV = ENV['INSTANCE_ID'] ? RestClient.get('http://nav.heroku.com/v1/providers/header') : nil
-    before do
-      headers['Expires'] = 'Sun, 19 Nov 1978 05:00:00 GMT'
-      headers['Cache-Control'] = 'no-store, no-cache, must-revalidate'  
-      headers['Pragma'] = 'no-cache'
-    end
-  end
-end
-
-Dir[File.join(File.dirname(__FILE__),"app/**/*.rb")].each do |file|
-  require file
-end
+puts "\nDEPRECATION WARNING: 'rhoconnect/console/server' will be removed in 4.0. Please change your config.ru:\n\n"
+puts "require 'rhoconnect/console/server'"
+puts "\nto the following:\n\n"
+puts "require 'rhoconnect/web-console/server'\n\n"
+require 'rhoconnect/web-console/server'

data/lib/rhoconnect/console.rb

@@ -1,3 +1,3 @@
-path = File.join(File.dirname(__FILE__),'..','..','lib')
+path = File.join(File.dirname(__FILE__), '..')
 $:.unshift path
-ROOT_PATH = File.expand_path(File.dirname(__FILE__))
+ROOT_PATH = Dir.pwd