cloudblocks 0.0.15 → 0.0.17

data/bin/cb-agent

@@ -2,13 +2,14 @@
 
 require 'yaml'
 require File.expand_path("../../lib/cloud-quartz", __FILE__)
+require File.expand_path("../../lib/client_auth", __FILE__)
 require 'optparse'
 require 'socket'
 require 'logger'
 require 'highline/import'
 begin
 	gem 'eventmachine', '~>1.0.0.beta.4'
-	gem 'faye', '~>0.8.0'
+	gem 'faye', '~>0.8.3'
 	gem 'highline', '~>1.6.11'
 
 	require 'eventmachine'
@@ -48,7 +49,7 @@ def start
 			end
 		end
 
-		check_version
+		#check_version
 		load_plugins
 	rescue => exc
 		@log.error exc.message
@@ -56,14 +57,14 @@ def start
 	end
 
 	begin
-		@log.info @quartz.status(1, @version, plugin_meta_data)
+		@log.info @quartz.init(@version, plugin_meta_data)
 	rescue Exception => e
 		message = e.message
 		if message =~ /Couldn't find Agent with uid =/
 			@log.warn "This agent is no longer registered at the server. The old registration details have been removed from this agent. Please re-run the agent to re-register it."
 			puts "This agent is no longer registered at the server. The old registration details have been removed from this agent. Please re-run the agent to re-register it."
 			puts @config_full
-		File.delete(@config_full)
+			File.delete(@config_full)
 		else
 			@log.error e.message
 		end
@@ -99,9 +100,9 @@ def stop
 	@log.info "Attempting to stop the CloudBlocks Agent"
 	puts "Attempting to stop the CloudBlocks Agent..."
 
-	begin
-		@quartz.status(2, @version, plugin_meta_data)
-		EM.stop
+	begin		
+		@quartz.status(2)
+		EM.stop		
 	rescue
 	end
 
@@ -118,11 +119,11 @@ def stop
 	end
 end
 
-def register
+def register(stack_uid = nil)
 	puts "Registering the CloudBlocks Agent with #{@url} and API Key #{@api_key}..."
 	os_name = RUBY_PLATFORM
 	os_id = os_name.include?('darwin') ? 5 : 1
-	agent = { :agent_type_id => os_id, :agent_name => Socket.gethostname, :agent_timezone => Time.new.zone, :extra => os_name}
+	agent = { :agent_type_id => os_id, :agent_name => Socket.gethostname, :agent_timezone => Time.new.zone, :extra => os_name, :stack_uid => stack_uid }
 
 	result = @quartz.register(agent)
 	if result['ok']
@@ -246,43 +247,50 @@ def run_plugin(plugin, message)
 	end
 end
 
-def check_version
+#def check_version
+#	begin
+#		result = @quartz.check_version
+#		if result['ok']
+#			latest = result['latest']
+#			@log.warn 'A newer version of the CloudBlocks Agent is available. Please update the cloudblocks gem. See http://help.cloudblocks.co for more info' if latest > @version
+#		end
+#	rescue => exc
+#		warn "Cannot connect to the CloudBlocks server"
+#		exit -1
+#	end
+#end
+
+def pulsate
+	@log.debug "Pulsate"
 	begin
-		result = @quartz.check_version
-		if result['ok']
-			latest = result['latest']
-			@log.warn 'A newer version of the CloudBlocks Agent is available. Please update the cloudblocks gem. See http://help.cloudblocks.co for more info' if latest > @version
-		end
+		@quartz.pulse 
 	rescue => exc
-		warn "Cannot connect to the CloudBlocks server"
-		exit -1
+		@log.error "Failed to pulsate due to #{exc.message}"
 	end
 end
 
 def run
 	EM.run{
-	 	Signal.trap('INT') { stop }
-	    Signal.trap('TERM'){ stop }
+		Signal.trap('INT') { stop }
+		Signal.trap('TERM'){ stop }
 
 	    # pulse
+	    pulsate
 		EM.add_periodic_timer 60 do  # 1 minute
-			@log.debug "Pulseate"
-			begin
-				@quartz.pulse 
-			rescue => exc
-				@log.error "Failed to pulsate due to #{exc.message}"
-			end
+			pulsate
 		end
 
-	    if @realtime
-	    	@log.info "Listening to realtime nofitifications from /quartz/agents/#{@agent_id} on #{@faye_url}" 
-	    	client = Faye::Client.new(@faye_url)
-			client.subscribe("/quartz/agents/#{@agent_id}") do |message|
-    			@log.info "Got realtime notice for a new job #{message}"
-    			get_job
+		if @realtime
+			@log.info "Listening to realtime nofitifications from /quartz/agents/#{@agent_id} on #{@faye_url}" 
+
+			client = Faye::Client.new(@faye_url)
+			client.add_extension(ClientAuth.new(@api_key, @secret_key))
+			client.subscribe("/user/#{@api_key}/quartz/agents/#{@agent_id}") do |message|
+				@log.info "Got realtime notice for a new job #{message}"
+				get_job
 			end
-	    else
-	    	@log.info "Checking for new jobs every 5 seconds"
+		else
+			@log.info "Checking for new jobs every 5 seconds"
 			EM.add_periodic_timer 5 do
 				get_job
 			end
@@ -304,10 +312,10 @@ end
 def pid_process_running?(pid)
 	begin
 		pid_number = pid.to_i
-  		Process.getpgid(pid_number)
-  		true
+		Process.getpgid(pid_number)
+		true
 	rescue Errno::ESRCH
-  		false
+		false
 	end
 end
 
@@ -329,7 +337,7 @@ Dir.mkdir(@cb_tmp_dir) if !File.exists?(@cb_tmp_dir)
 @log_full = File.join(@cb_tmp_dir, @log_file)
 
 @api_key = ''
-@faye_url = 'https://sockets.cloudblocks.co/push/'
+@faye_url = 'https://sockets.cloudblocks.co/push'
 commands = %w[register unregister start stop]
 
 @plugins = {}
@@ -341,13 +349,13 @@ command = nil
 @realtime = true
 OptionParser.new do |opts|
 	opts.banner = 	<<-EOF
-CloudBlocks Agent. v#{str_version} (c) 2012 CloudBlocks
-For more information please visit http://cloudblocks.co
+	CloudBlocks Agent. v#{str_version} (c) 2012 CloudBlocks
+	For more information please visit http://cloudblocks.co
 
-Usage: cb-agent [register|unregister|start|stop] [options]
+	Usage: cb-agent [register|unregister|start|stop] [options]
 
-Options:
-EOF
+	Options:
+	EOF
 
 	opts.on('--url URL', 'Server URL') do |server_url|
 		@url = server_url
@@ -376,7 +384,6 @@ EOF
 
 	opts.on('--sockets SOCKETS', 'Sockets URL') do |v|
 		@faye_url = v
-		@faye_url = "#{@faye_url}/" if @faye_url[-1] != '/'
 	end
 
 	opts.on('--api-key APIKEY', 'API key') do |v|
@@ -387,6 +394,10 @@ EOF
 		@secret_key = v
 	end
 
+	opts.on('--stack STACKIUD', 'Stack id') do |v|
+		@stack_uid = v
+	end
+
 	opts.on('-r', '--realtime', 'Enable realtime notifications (default)') do |v|
 		@realtime = true
 	end
@@ -399,11 +410,11 @@ EOF
 		puts opts
 		puts <<-EOF
 
-Commands:
-	register 			Register the CloudBlocks Agent
-	start 				Starts the CloudBlocks Agent as a deamon
-	stop 				Stops the CloudBlocks Agent daemon
-	unregister			Unregisters the CloudBlocks Agent
+		Commands:
+		register 			Register the CloudBlocks Agent
+		start 				Starts the CloudBlocks Agent as a deamon
+		stop 				Stops the CloudBlocks Agent daemon
+		unregister			Unregisters the CloudBlocks Agent
 
 		EOF
 		exit 0
@@ -447,7 +458,7 @@ end
 # no agent id?
 if @agent_id.empty?
 	@quartz = CloudQuartz.new(:api_key => @api_key, :url => @url, :secret_key => @secret_key)
-	register 
+	register(@stack_uid) 
 	@quartz = CloudQuartz.new(:api_key => @api_key, :url => @url, :agent_id => @agent_id, :secret_key => @secret_key)
 	start
 end

data/lib/cloud-quartz.rb

@@ -14,44 +14,41 @@ class CloudQuartz
 	end
 
 	def get_job
-		process(self.class.get("/queue/#{@agent_id}.json", { :headers => http_headers } ))
+		process(self.class.get("/queue/#{@agent_id}.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key) } ))
 	end
 
 	def register(agent)
-		process(self.class.post('/agent.json', { :headers => http_headers.merge({'Content-Type' => 'application/json'}), :body => agent.to_json }))
+		process(self.class.post('/agent.json', { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => agent.to_json }))
 	end
 
 	def unregister(agent)
-		process(self.class.delete("/agent/#{@agent_id}.json", :headers => http_headers))
+		process(self.class.delete("/agent/#{@agent_id}.json", :headers => ClientAuth.build_headers(@api_key, @secret_key)))
 	end
 
-	def check_version
-		self.class.get("/agent/version", :headers => http_headers)
-	end
+	#TODO: Is this deprecated now?
+	#def check_version
+	#	self.class.get("/agent/version", :headers => ClientAuth.build_headers(@api_key, @secret_key))
+	#end
 
 	def post_results(job_id, data)
-		process(self.class.post("/job/#{job_id}/complete.json", { :headers => http_headers.merge({'Content-Type' => 'application/json'}), :body => data.to_json } ))
+		process(self.class.post("/job/#{job_id}/complete.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json } ))
 	end
 
 	def pulse
-		process(self.class.get("/agent/#{@agent_id}/pulse.json", { :headers => http_headers } ))
+		process(self.class.get("/agent/#{@agent_id}/pulse.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key) } ))
 	end
 
-	def status(stat, version, plugins)
-		data = { :status => stat, :version => version, :plugins => plugins }
-		process(self.class.post("/agent/#{@agent_id}/status.json", { :headers => http_headers.merge({'Content-Type' => 'application/json'}), :body => data.to_json }))
+	def status(stat)
+		data = { :status => stat }
+		process(self.class.post("/agent/#{@agent_id}/status.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json }))
 	end
 
-	private
-
-	def http_headers
-		time = Time.now.utc.to_i
-		{ 'api_key' => @api_key, 'hash' => signature(time), 'time' => time.to_s }
+	def init(version, plugins)
+		data = { :version => version, :plugins => plugins }
+		process(self.class.post("/agent/#{@agent_id}/initialize.json", { :headers => ClientAuth.build_headers(@api_key, @secret_key).merge({'Content-Type' => 'application/json'}), :body => data.to_json }))
 	end
 
-	def signature(time)
-		Digest::SHA1.hexdigest("#{@api_key}#{@secret_key}#{time}").downcase
-	end
+	private
 
 	def process(response)
 		if response.code != 200

data/lib/plugins/backup.rb

@@ -0,0 +1,74 @@
+require File.join(File.dirname(__FILE__), 'quartz_plugin')
+require 'tempfile'
+require 'fileutils'
+
+class Backup < QuartzPlugin
+	
+	@@version_major = 0
+	@@version_minor = 0
+	@@version_revision = 1
+
+	def info
+		{ :uid => "64cec894b98b43ce9e6047a9284a4b7d", :name => "Backup", :version => get_version }
+	end
+
+	def run(message)
+		
+		pl = payload(message)
+
+		job_name = pl['job_name']
+		backup_script = "#Script generated by Cloud66 job '#{job_name}'\n\n" + pl['script']
+		script_name = pl['script_name']
+
+		random_file = Tempfile.new('cloud66').path + '.rb'
+    	File.open(random_file, 'w') do |f|  
+      		f.puts backup_script
+    	end
+		
+		backup_root_directory = File.expand_path('~/.cloud66/backup')
+		FileUtils.mkdir_p(backup_root_directory)
+		
+		command = "backup perform --trigger #{script_name} --root-path #{backup_root_directory} --config_file #{random_file} 2>&1"
+		@log.info "Shell command '#{command}'"
+
+		begin
+			result = run_shell("#{command}")
+			data = 	result[:message]
+		
+			#regex to identify all the "error" lines
+			errorRegex = Regexp.new('(?<line>\[\d{4}\/\d{2}\/\d{2}\s\d{2}\:\d{2}\:\d{2}\]\[\\e\[31merror\\e\[0m\].*?)(\\n|$)')
+			errors = data.scan(errorRegex)
+			
+			if errors.size > 0
+				
+				#log all the errors
+				@log.info errors
+
+				#get rid of the backtrace
+				removeRegex = Regexp.new('(Backtrace\:|\d{2}\:\d{2}\]\[\\e\[31merror\\e\[0m\]\s*\/)')
+
+				#subregex to get rid of nasty timestamp and [\e[31merror\e[0m]
+				replaceRegex = Regexp.new('^.*\[\\e\[31merror\\e\[0m\]')
+				
+				errorResult = []
+				errors.each do |errorArr|
+					error = errorArr[0]
+					errorResult << error.gsub(replaceRegex,'').chomp unless error =~ removeRegex
+				end
+
+				run_result(false, errorResult.join("\n"))
+			else
+				if (result[:ok])
+					run_result(true, "Backup completed successfully!")
+				else
+					run_result(true, "An error occurred!")
+				end
+			end
+
+		rescue => ex
+			run_result(false, "Failed to execute backup due to #{ex}")
+		ensure 
+			File.delete random_file if File.exists? random_file
+		end
+	end
+end	

data/lib/plugins/s3_backup.rb

@@ -21,7 +21,7 @@ class S3Backup < QuartzPlugin
 		@secret_key 			= pl['secret_key']
 		@bucket 				= pl['bucket']
 		@region 				= pl['region']
-		@remote_path 		= pl['remote_path']
+		@remote_path 			= pl['remote_path']
 		@local_pattern 			= pl['local_pattern']
 		@keep 					= pl['keep'].empty? ? 0 : pl['keep'].to_i
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloudblocks
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.17
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-10 00:00:00.000000000 Z
+date: 2012-08-14 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506935040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,15 +21,10 @@ dependencies:
         version: 0.8.1
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.8.1
+  version_requirements: *70279506935040
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506933860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -37,15 +32,10 @@ dependencies:
         version: 1.6.3
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.6.3
+  version_requirements: *70279506933860
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506933120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -53,15 +43,10 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.12.10
+  version_requirements: *70279506933120
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506932280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -69,15 +54,10 @@ dependencies:
         version: 1.0.0.beta.4
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.0.0.beta.4
+  version_requirements: *70279506932280
 - !ruby/object:Gem::Dependency
   name: faye
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506931560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -85,15 +65,10 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.8.0
+  version_requirements: *70279506931560
 - !ruby/object:Gem::Dependency
   name: open4
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506930800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -101,53 +76,49 @@ dependencies:
         version: 1.3.0
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.3.0
+  version_requirements: *70279506930800
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70279506913940 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: 1.4.0
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.1.2
+  version_requirements: *70279506913940
 - !ruby/object:Gem::Dependency
-  name: highline
-  requirement: !ruby/object:Gem::Requirement
+  name: cloud66-backup
+  requirement: &70279506913160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.11
+        version: 3.0.25
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  version_requirements: *70279506913160
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: &70279506912320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.6.11
+  type: :runtime
+  prerelease: false
+  version_requirements: *70279506912320
 description: See http://cloudblocks.co for more info
-email: hello@thecloudblocks.com
+email: hello@cloudblocks.co
 executables:
 - cb-agent
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/config-chief.rb
 - lib/cloud-quartz.rb
+- lib/plugins/backup.rb
 - lib/plugins/broken.rb
 - lib/plugins/file_rotate.rb
 - lib/plugins/log_rotate.rb
@@ -181,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: CloudBlocks Gem and Agent

data/lib/config-chief.rb

@@ -1,100 +0,0 @@
-require 'HTTParty'
-require 'socket'
-require 'json'
-
-class ConfigChief
-  include HTTParty
-
-  @api_key = ""
-  @workspace = ""
-  
-  def initialize(options = {})
-    @api_key = options[:api_key]
-    @workspace = options[:workspace]
-    self.class.base_uri options[:url] || 'https://api.cloudblocks.co'
-  end
-  
-  def get_by_key(key, options = {}, params = {})
-    opts = {
-      :default_value => nil,
-      :full_obj => false
-    }.merge(options)
-    
-    optionals = {}
-    params.each do |k, v|
-      optionals["cc-#{k}"] = v
-    end
-    
-    headers = { :headers => http_headers.merge(optionals) }
-    
-    result = self.class.get("/workspaces/#{@workspace}/value.json", headers.merge(:query => { :query => key } ))
-    
-    if result.response.code.to_i == 200
-      retrieved = result.parsed_response 
-      return retrieved if opts[:full_obj]
-      return retrieved['parsed_value']
-    end
-    
-    return opts[:default_value]
-  end
-
-  def get_by_id(id, options = {}, params = {})
-    opts = {
-      :default_value => nil,
-      :full_obj => false
-    }.merge(options)
-    
-    optionals = {}
-    params.each do |k, v|
-      optionals["cc-#{k}"] = v
-    end
-    
-    headers = { :headers => http_headers.merge(optionals) }
-    
-    result = self.class.get("/workspaces/#{@workspace}/config_keys/#{id}.json", headers )
-    
-    if result.response.code.to_i == 200
-      retrieved = result.parsed_response 
-      return retrieved if opts[:full_obj]
-      return retrieved['parsed_value']
-    end
-    
-    return opts[:default_value]
-  end
-  
-  def workspaces
-    self.class.get("/workspaces.json", { :headers => http_headers } )
-  end
-  
-  def config_items(query = '*')
-    headers = { :headers => http_headers }
-    self.class.get("/workspaces/#{@workspace}/config_keys.json", headers.merge(:query => { :query => query } ))
-  end
-  
-  def set_value(key, value)
-    headers = { :headers => http_headers.merge({'Content-Type' => 'application/json'}) }
-    self.class.post("/workspaces/#{@workspace}/config_keys.json", headers.merge(:body => { :key => key, :value => value }.to_json))
-  end
-  
-  def register_node(options = {})
-    headers = { :headers => http_headers.merge({'Content-Type' => 'application/json'}) }
-    self.class.post("/workspaces/#{@workspace}/node.json", headers.merge(:body => options.to_json))
-  end
-
-  def unregister_node(node_uid)
-    headers = { :headers => http_headers }
-    self.class.delete("/workspaces/#{@workspace}/node/#{node_uid}.json", headers)
-  end
-  
-  def update_node_status(node_uid, status)
-    headers = { :headers => http_headers.merge({'Content-Type' => 'application/json'}) }
-    self.class.post("/workspaces/#{@workspace}/node/#{node_uid}/status.json", headers.merge({ :body => { :status => status}.to_json}))
-  end
-  
-  private 
-  
-  def http_headers
-    { 'api_key' => @api_key, 'ConfigChief-Node' => Socket.gethostname }
-  end
-  
-end