rhoconnect 3.1.2 → 3.2.0.beta1

data/lib/rhoconnect/graph_helper.rb

@@ -0,0 +1,225 @@
+module GraphHelper
+  
+   def source_timing(params)
+      #name,data,options
+      puts "display is #{params}"
+      @displayname = params['display_name']
+      names = []
+      begin
+        names = get_sources('all') #RhoconnectApi::list_sources(session[:server],session[:token],:all)
+      rescue Exception => e
+        puts "errror #{e.message}"
+      end
+      @sources = []
+      
+      names.each do |name|
+        s = {}
+        data = []
+        series = []
+        options = { :legend => { :show => true }, :title => name }
+        s['name'] = name
+
+        keys = get_user_count("source:*:#{name}") #RhoconnectApi::stats(session[:server],session[:token], :names => "source:*:#{name}")
+
+        xmin = 9999999999999999
+        xmax = -1
+        ymin = 9999999999999999
+        ymax = -1
+        keys.each do |key|
+          method = key.gsub(/source:/,"").gsub(/:.*/,"")
+          series << {:showLabel => true, :label => method }
+          
+          range = get_user_count(nil,key,0,-1) #RhoconnectApi::stats(session[:server],session[:token], {:metric => key, :start => 0, :finish => -1})
+          thisdata = []
+          range.each do |value|
+            count = value.split(',')[0]
+            value.gsub!(/.*,/,"")
+            thisdata << value.split(":").reverse
+            thisdata[-1][0] = thisdata[-1][0].to_i * 1000
+            thisdata[-1][1] = thisdata[-1][1].to_f
+            thisdata[-1][1] /= count.to_f
+
+            ymin = thisdata[-1][1].to_f if thisdata[-1][1] && thisdata[-1][1].to_f < ymin
+            ymax = thisdata[-1][1].to_f if thisdata[-1][1] && thisdata[-1][1].to_f > ymax
+
+          end
+          data << thisdata
+          xmin = thisdata[0][0].to_i if thisdata[0] && thisdata[0][0].to_i < xmin
+          xmax = thisdata[-1][0].to_i if thisdata[-1]  && thisdata[-1][0].to_i > xmax
+        end
+        
+        options[:axes] = {
+          :yaxis => { :tickOptions => { :formatString =>'%.3f'}, :autoscale => true, :min => 0, :max => ymax + (ymax * 0.05), :label  => 'Seconds', :labelRenderer => '$.jqplot.CanvasAxisLabelRenderer'  }, 
+          :xaxis => { :autoscale => true, :renderer=>'$.jqplot.DateAxisRenderer',
+            :tickOptions => {:formatString => '%m/%d/%y'}}
+        }
+
+        s['data'] = data
+        options[:series] = series
+        options[:cursor] = {:zoom => true, :showTooltip => true} 
+
+        s['options'] = options
+
+        @sources << s
+      end
+      @graph_t = 'source'
+      @data = [[[1,2],[3,4],[5,6]]].to_json
+      erb :jqplot, :layout => false
+    end
+
+
+   def http_timing(params)
+      @uri   = 'timing/httptiming'
+
+      #name,data,options
+      @displayname = params['display_name']
+      names = ["GET","POST"]
+      begin
+        names = get_sources('all') #RhoconnectApi::list_sources(session[:server],session[:token],:all)
+      rescue Exception => e
+        puts "errror #{e.message}"
+      end
+      names << "ALL"
+      @sources = []
+
+      names.each do |name|
+        s = {}
+        data = []
+        series = []
+        options = { :legend => { :show => true }, :title => name }
+        s['name'] = name
+
+        name = "*" if name == "ALL"
+        keys = get_user_count("http:*:#{name}") #RhoconnectApi::stats(session[:server],session[:token], :names => "http:*:#{name}")
+
+        xmin = 9999999999999999
+        xmax = -1
+        ymin = 9999999999999999
+        ymax = -1
+        keys.each do |key|
+          method = key.gsub(/http:.*?:/,"")
+          method.gsub!(/:.*/,"") unless name == "*"
+          series << {:showLabel => true, :label => method }
+          
+          range = get_user_count(nil,key,0,-1) #RhoconnectApi::stats(session[:server],session[:token], {:metric => key, :start => 0, :finish => -1})
+          thisdata = []
+          range.each do |value|
+            count = value.split(',')[0]
+            value.gsub!(/.*,/,"")
+            thisdata << value.split(":").reverse
+            thisdata[-1][0] = thisdata[-1][0].to_i * 1000
+            thisdata[-1][1] = thisdata[-1][1].to_f
+            thisdata[-1][1] /= count.to_f
+
+            ymin = thisdata[-1][1].to_f if thisdata[-1][1] && thisdata[-1][1].to_f < ymin
+            ymax = thisdata[-1][1].to_f if thisdata[-1][1] && thisdata[-1][1].to_f > ymax
+
+          end
+          data << thisdata
+          xmin = thisdata[0][0].to_i if thisdata[0] && thisdata[0][0].to_i < xmin
+          xmax = thisdata[-1][0].to_i if thisdata[-1]  && thisdata[-1][0].to_i > xmax
+        end
+
+        options[:axes] = {
+          :yaxis => { :tickOptions => { :formatString =>'%.3f'}, :autoscale => true, :min => 0, :max => ymax + (ymax * 0.05), :label  => 'Seconds', :labelRenderer => '$.jqplot.CanvasAxisLabelRenderer'  }, 
+          :xaxis => { :autoscale => true, :renderer=>'$.jqplot.DateAxisRenderer',
+            :tickOptions => {:formatString => '%m/%d/%y'}}
+        }
+
+        s['data'] = data
+        options[:series] = series
+        options[:cursor] = {:zoom => true, :showTooltip => true} 
+
+        s['options'] = options
+
+        @sources << s
+      end
+      @graph_t = 'http'
+      @data = [[[1,2],[3,4],[5,6]]].to_json
+      erb :jqplot, :layout => false
+    end
+  
+  def count_graph(uri,title,name,metric)
+  	start = 0
+  	finish = -1
+  	now = Time.now.to_i
+  	format = "%m/%d/%y"
+
+  	thisdata = []
+  	series = []
+  	series << 
+  	options = { :legend => { :show => false },  :title => title }
+  	@sources = []
+
+  	s = {}
+  	usercount = []
+
+    usercount = get_user_count(nil,metric,start,finish)
+    
+  	usercount.each do |count|
+  	  user,timestamp = count.split(':')
+  	  user = user.to_i
+  	  timestamp = timestamp.to_i * 1000
+  	  thisdata << [timestamp,user]
+  	end
+
+  	  options[:axes] = {
+  	     :xaxis => { :autoscale => true, :renderer=>'$.jqplot.DateAxisRenderer',
+  	       :tickOptions => {:formatString => format}},
+  	     :yaxis => {:label  => name, :labelRenderer => '$.jqplot.CanvasAxisLabelRenderer'}
+  	   }
+
+  	options[:cursor] = {:zoom => true, :showTooltip => true} 
+  	s['name'] = name
+  	s['data'] = [thisdata]
+  	s['options'] = options
+  	@sources << s
+  	erb :jqplot, :layout => false
+  end
+  
+  def get_user_count(name=nil,metric_p=nil,start=nil,finish=nil)
+   begin
+       if Rhoconnect.stats == true
+           names = name
+           if names
+             Rhoconnect::Stats::Record.keys(names)
+           else
+             metric = metric_p
+             rtype = Rhoconnect::Stats::Record.rtype(metric)
+             if rtype == 'zset'
+               # returns [] if no results
+               Rhoconnect::Stats::Record.range(metric,start,finish)
+             elsif rtype == 'string'
+               Rhoconnect::Stats::Record.get_value(metric) || ''
+             else
+               raise ApiException.new(404, "Unknown metric")
+             end
+           end
+       else
+         raise ApiException.new(500, "Stats not enabled")
+       end
+     rescue Exception => e
+       puts "error #{e.message} -- #{metric}"
+       usercount = ["0:#{Time.now.to_i}"]
+     end
+  end
+  
+  def get_sources(partition_type)
+    sources = App.load(APP_NAME).sources
+    puts "sources is -- #{sources}"
+    if partition_type.nil? or partition_type == 'all'
+      sources
+    else
+      res = []
+      sources.each do |name|
+        s = Source.load(name,{:app_id => APP_NAME,:user_id => '*'})
+        if s.partition_type and s.partition_type == partition_type.to_sym
+          res << name 
+        end
+      end  
+      res
+    end
+    
+  end
+
+end

data/lib/rhoconnect/jobs/bulk_data_job.rb

@@ -20,19 +20,39 @@ module Rhoconnect
           create_hsql_data_file(bulk_data,ts) if Rhoconnect.blackberry_bulk_sync
           lap_timer('create_hsql_data_file',timer)
           log "finished bulk data process"
+          #bulk_data.state = :completed
+          #bulk_data.refresh_time = Time.now.to_i + Rhoconnect.bulk_sync_poll_interval
+        else
+          raise Exception.new("No bulk data found for #{params["data_name"]}")
+        end
+      rescue Exception => e
+        bulk_data.delete if bulk_data
+        log "Bulk data job raised: #{e.message}"
+        log e.backtrace.join("\n")
+        raise e
+      end
+    end
+
+    def self.after_perform_x(*args)
+      log "BulkDataJob.after_perform_x hook called ..."
+      params = args[0] # 1st parameter is bulk data
+      begin
+        bulk_data = BulkData.load(params["data_name"]) if BulkData.is_exist?(params["data_name"])
+        if bulk_data
           bulk_data.state = :completed
           bulk_data.refresh_time = Time.now.to_i + Rhoconnect.bulk_sync_poll_interval
+          log "BulkDataJob.after_perform_x hook set data state to complete."
         else
           raise Exception.new("No bulk data found for #{params["data_name"]}")
         end
       rescue Exception => e
         bulk_data.delete if bulk_data
-        log "Bulk data job raised: #{e.message}"
+        log "Bulk data after_perform_x raised: #{e.message}"
         log e.backtrace.join("\n")
         raise e
       end
     end
-    
+
     def self.import_data_to_object_values(db,source)
       data = source.get_data(:md)
       counter = {}
@@ -129,6 +149,8 @@ module Rhoconnect
       sources_refs = {}
       schema,index,bulk_data.dbfile = get_file_args(bulk_data.name,ts)
       FileUtils.mkdir_p(File.dirname(bulk_data.dbfile))
+      # TODO: remove old bulk files!
+      # FileUtils.rm Dir.glob(File.join(Rhoconnect.data_directory, "#{bulk_data.name}*"))
 
       db = DBAdapter.instance.get_connection(bulk_data.dbfile)    
       db.execute_batch(File.open(schema,'r').read)

data/lib/rhoconnect/server.rb

@@ -3,7 +3,6 @@ require 'sinatra/base'
 require 'erb'
 require 'json'
 require 'fileutils'
-#require 'rack/contrib'
 require 'rhoconnect'
 require 'rhoconnect/x_domain_session_wrapper'
 require 'rhoconnect/body_content_type_parser'
@@ -20,9 +19,6 @@ module Rhoconnect
   end
     
   class Server < Sinatra::Base
-    libdir = File.dirname(File.expand_path(__FILE__))
-    set :views,         "#{libdir}/server/views"
-    set :public_folder, "#{libdir}/server/public"
     set :static,        true
     set :stats,         false
     # default secret
@@ -200,7 +196,30 @@ module Rhoconnect
       end
     end
     
+    private 
+    def self._use_async_framework
+      return false if settings.respond_to?(:use_async_model) and settings.use_async_model == false
+      return false if @dispatch_framework_initialized
+      
+      @dispatch_framework_initialized ||=true
+      if RUBY_VERSION =~ /1.9/ and not defined?(JRUBY_VERSION)
+        require 'rhoconnect/async' 
+        register Rhoconnect::Synchrony
+        helpers Rhoconnect::AsyncHelpers
+      else
+        settings.use_async_model = false
+      end
+    end
+    
     def self.new
+      # by default, enable this feature
+      if not settings.respond_to?(:use_async_model) or settings.use_async_model != false
+        set :use_async_model, true
+      end
+      # this must be called first - because
+      # it redefines some of the middleware
+      Rhoconnect::Server._use_async_framework
+      
       if settings.respond_to?(:stats) and settings.send(:stats) == true
         Rhoconnect.stats = true
       else
@@ -208,6 +227,7 @@ module Rhoconnect
         Rhoconnect.stats = false
       end
       settings.use_middleware
+      
       super
     end
         
@@ -222,12 +242,16 @@ module Rhoconnect
     before do
       cache_control :no_cache
       headers({'pragma'=>'no-cache'})
-
+      
       if params[:version] and params[:version].to_i < 3
         throw :halt, [404, "Server supports version 3 or higher of the protocol."]
       end
     end
-
+    
+    get '/' do
+      redirect "/console/"
+    end
+    
     %w[get post].each do |verb|
       send(verb, "/*application*") do
         unless request_action == 'clientlogin'
@@ -236,11 +260,7 @@ module Rhoconnect
         pass
       end
     end
-
-    get '/' do
-      erb :index
-    end
-
+    
     # old routes
     post '/login' do
       mark_deprecated_call_and_reroute(:login, :admin, self, params)

data/lib/rhoconnect/source.rb

@@ -68,6 +68,13 @@ module Rhoconnect
       end
     end
     
+    def update_fields(fields)
+      fields.each do |name,value|
+        arg = "#{name}=".to_sym 
+        self.send(arg, value)
+      end
+    end
+    
     def to_array
       res = []
       @@string_fields.each do |field|
@@ -90,14 +97,11 @@ module Rhoconnect
     
     # source fields
     define_fields([:id, :rho__id, :name, :url, :login, :password, :callback_url, :partition_type, :sync_type, 
-      :queue, :query_queue, :cud_queue, :belongs_to, :has_many, :pass_through], [:source_id, :priority, :retry_limit])
+      :queue, :query_queue, :cud_queue, :belongs_to, :has_many, :pass_through], [:source_id, :priority, :retry_limit, :simulate_time])
     
     def initialize(fields)
       self.name = fields['name'] || fields[:name]
-      fields.each do |name,value|
-        arg = "#{name}=".to_sym 
-        self.send(arg, value) if self.respond_to?(arg)
-      end
+      update_fields(fields)
     end
   
     def self.set_defaults(fields)
@@ -113,6 +117,7 @@ module Rhoconnect
       fields[:belongs_to] = fields[:belongs_to].to_json if fields[:belongs_to]
       fields[:schema] = fields[:schema].to_json if fields[:schema]
       fields[:retry_limit] = fields[:retry_limit] ? fields[:retry_limit] : 0
+      fields[:simulate_time] = fields[:simulate_time] ? fields[:simulate_time] : 0
     end
         
     def self.create(fields,params)

data/lib/rhoconnect/source_sync.rb

@@ -111,6 +111,29 @@ module Rhoconnect
         @source.app_id,@source.user_id,params)
     end
     
+    def fast_insert(new_objs, timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        diff_count = new_objs.size
+        @source.put_data(:md, new_objs, true)
+        @source.update_count(:md_size,diff_count)
+      end
+    end
+    
+    def fast_update(orig_hash, new_hash, timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        @source.delete_data(:md, orig_hash)
+        @source.put_data(:md, new_hash, true)
+      end
+    end 
+    
+    def fast_delete(delete_objs, timeout=10,raise_on_expire=false)
+      @source.lock(:md,timeout,raise_on_expire) do |s|
+        diff_count = -delete_objs.size
+        @source.delete_data(:md, delete_objs)
+        @source.update_count(:md_size,diff_count)
+      end
+    end
+    
     def push_objects(objects,timeout=10,raise_on_expire=false,rebuild_md=true)
       @source.lock(:md,timeout,raise_on_expire) do |s|
         diff_count = 0
@@ -360,10 +383,17 @@ module Rhoconnect
     # Metadata Operation; source adapter returns json
     def _get_data(method)
       if @adapter.respond_to?(method)
-        data = @adapter.send(method) 
+        data = @adapter.send(method)        
         if data
           @source.put_value(method,data)
-          @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
+          if method == :schema
+            parsed = JSON.parse(data)
+            schema_version = parsed['version']
+            raise "Mandatory version key is not defined in source adapter schema method" if schema_version.nil?
+            @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(schema_version))
+          else
+            @source.put_value("#{method}_sha1",Digest::SHA1.hexdigest(data))
+          end          
         end
       end
     end

data/lib/rhoconnect/tasks.rb

@@ -1,111 +1,19 @@
-require 'json'
 require 'zip/zip'
 require 'uri'
-require File.join(File.dirname(__FILE__),'console','rhoconnect_api')
+$:.unshift File.dirname(__FILE__)
+require 'console/rhoconnect_api'
+require 'utilities'
 
+DEPRECATION_MSG =  "\n" + 
+                   "*** Warning:  As of version 4.0 of rhoconnect, the rake command will be deprecated. ***\n" +
+                   "*** Use the corresponding rhoconnect commands instead.                              ***\n" +
+                   "*** Type 'rhoconnect help' for more information on these commands.                  ***\n" +
+                   "\n"
 
-module Rhoconnect
-  module TaskHelper
-    def post(path,params)
-      req = Net::HTTP.new($host,$port)
-      resp = req.post(path, params.to_json, 'Content-Type' => 'application/json')
-      print_resp(resp, resp.is_a?(Net::HTTPSuccess) ? true : false)
-    end
-
-    def print_resp(resp,success=true)
-      if success
-        puts "=> OK" 
-      else
-        puts "=> FAILED"
-      end
-      puts "=> " + resp.body if resp and resp.body and resp.body.length > 0
-    end
-
-    def archive(path)
-      File.join(path,File.basename(path))+'.zip'
-    end
-
-    def ask(msg)
-      print msg
-      STDIN.gets.chomp
-    end
-    
-    def load_settings(file)
-      begin
-        $settings = YAML.load_file(file)
-      rescue Exception => e
-        puts "Error opening settings file #{file}: #{e}."
-        puts e.backtrace.join("\n")
-        raise e
-      end
-    end
-    
-    def rhoconnect_socket
-      '/tmp/rhoconnect.dtach'
-    end
-    
-    def rhoconnect_pid
-      if windows?
-    	  Dir.mkdir 'C:/TMP' unless File.directory? 'C:/TMP'
-    	  'C:/TMP/rhoconnect.pid'
-      else
-        '/tmp/rhoconnect.pid'
-      end
-    end
-
-    def windows?
-      RUBY_PLATFORM =~ /(win|w)32$/
-    end
-    
-    def ruby19?
-      RUBY_VERSION =~ /1.9/ 
-    end
-
-    def jruby?
-      defined?(JRUBY_VERSION)
-    end
-      
-    def thin?
-      begin
-        require 'thin'
-        'rackup -s thin'
-      rescue LoadError
-        nil
-      end
-    end
-    
-    def mongrel?
-      begin
-        require 'mongrel'
-        'rackup -s mongrel'
-      rescue LoadError
-        nil
-      end
-    end
-    
-    def report_missing_server
-      msg =<<-EOF
-Could not find 'thin' or 'mongrel' on your system.  Please install one:
-  gem install thin
-or
-  gem install mongrel
-EOF
-      puts msg
-      exit 1
-    end
-
-#    def jetty_rackup?
-#      'jruby -S jetty-rackup'
-#    end
-
-    def trinidad?
-      'jruby -S trinidad -p 9292 -r config.ru'
-    end
-  end
-end
+puts DEPRECATION_MSG if $0.include? 'rake'
 
 namespace :rhoconnect do
-  include Rhoconnect::TaskHelper
+  include Utilities
   include RhoconnectApi
   
   task :config do
@@ -256,7 +164,7 @@ namespace :rhoconnect do
     else
       puts 'Detach with Ctrl+\  Re-attach with rake rhoconnect:attach'
       sleep 2
-      sh "dtach -A #{rhoconnect_socket} #{cmd} config.ru -P #{rhoconnect_pid}"
+      system "dtach -A #{rhoconnect_socket} #{cmd} config.ru -P #{rhoconnect_pid}"
     end
   end
   
@@ -305,10 +213,13 @@ namespace :rhoconnect do
   
   desc "run rhoconnect console"
   task :console, :environment do |t, args|
-    if RedisRunner.running?        
+    if RedisRunner.running?
+      application_file = ruby19? ? './application' : 'application'
       #load development environment by default
       ENV['RACK_ENV'] = args[:environment] || 'development'
-      sh "irb -rubygems -r #{File.join(File.dirname(__FILE__),'console')} -r #{File.dirname(__FILE__)} -r application"
+      system "irb -rubygems -r #{File.join(File.dirname(__FILE__),'console')} " +
+             "-r #{File.dirname(__FILE__)} " +
+             "-r #{application_file}"
     else
       puts "Redis is not running. Please start it by running 'rake redis:start' command."
     end
@@ -319,13 +230,13 @@ namespace :rhoconnect do
     if windows?
   	  File.delete "#{rhoconnect_pid}" if system("FOR /F %A in (#{rhoconnect_pid}) do taskkill /F /PID %A")
     else
-      sh "cat #{rhoconnect_pid} | xargs kill -3"
+      system "cat #{rhoconnect_pid} | xargs kill -3"
     end
   end
   
   desc "Attach to rhoconnect console"
   task :attach => :dtach_installed do
-    sh "dtach -a #{rhoconnect_socket}" unless windows?
+    system "dtach -a #{rhoconnect_socket}" unless windows?
   end
   
   desc "Launch the web console in a browser - uses :syncserver: in settings.yml"
@@ -380,7 +291,7 @@ namespace :rhoconnect do
       end
       # build the executable WAR using the config/warble.rb file
       ENV['BUNDLE_WITHOUT'] = ['development','test'].join(':')
-      sh 'warble executable war'
+      system 'warble executable war'
     else
       puts "Cannot build WAR files outside of JRuby environment!"
     end

data/lib/rhoconnect/test_methods.rb

@@ -45,6 +45,26 @@ module Rhoconnect
       return @s.is_pass_through? ? res : md
     end
     
+    # Executes the adapter's search method based on search_params 
+    # and returns a hash of hashes.
+    # For example, if master document was 
+    #   {
+    #     "1"=>{"name"=>"Acme", "industry"=>"Electronics"},
+    #     "2"=>{"name"=>"Best", "industry"=>"Software"}
+    #   }
+    #
+    # Then call test_search("name"=>"Acme") would return 
+    #   {
+    #     "1"=>{"name"=>"Acme", "industry"=>"Electronics"},
+    #   }
+    def test_search(search_params)
+      pass_through = @s.pass_through
+      @s.pass_through = 'true'
+      res = @ss.search(@c.id, search_params)
+      @s.pass_through = pass_through
+      res
+    end
+
     # Executes the adapter's query method and returns 
     # the metadata stored in redis
     # For example, if your source adapter metadata method was: