CloudyScripts 1.10.44 → 2.11.45

data/Rakefile

@@ -12,7 +12,7 @@ require 'rake/testtask'
 
 spec = Gem::Specification.new do |s|
   s.name = 'CloudyScripts'
-  s.version = '1.10.44'
+  s.version = '2.11.45'
   s.has_rdoc = true
   s.extra_rdoc_files = ['README.rdoc', 'LICENSE']
   s.summary = 'Scripts to facilitate programming for infrastructure clouds.'

data/lib/help/script_execution_state.rb

@@ -1,11 +1,12 @@
 require "help/state_transition_helper"
+require "help/v_cloud_transition_helper"
 
 # Implements a little state-machine.
 # Usage: for every state you need, extend this class.
 # The method enter() must be implemented for every state you code and
 # return another state.
 class ScriptExecutionState
-  include StateTransitionHelper
+  include StateTransitionHelper, VCloudTransitionHelper
   
   # context information for the state (hash)
   attr_reader :context, :logger

data/lib/help/state_transition_helper.rb

@@ -637,7 +637,7 @@ module StateTransitionHelper
   # Get root filesytem type
   def get_root_partition_fs_type()
     post_message("Retrieving '/' root partition filesystem type...")
-    @logger.debug "get root partition filesystel type"
+    @logger.debug "get root partition filesystem type"
     # get root device and then its fs type
     root_fs_type = remote_handler().get_root_fs_type()
     @logger.debug "Found '#{root_fs_type}' as root filesystem type"
@@ -652,7 +652,7 @@ module StateTransitionHelper
   # Get root filesytem type and label
   def get_root_partition_fs_type_and_label()
     post_message("Retrieving '/' root partition filesystem type and label...")
-    @logger.debug "get root partition filesystel type"
+    @logger.debug "get root partition filesystem type"
     # get root device and then its fs type
     root_fs_type = remote_handler().get_root_fs_type()
     @logger.debug "Found '#{root_fs_type}' as root filesystem type"
@@ -681,7 +681,7 @@ module StateTransitionHelper
   # Get partition filesytem type
   def get_partition_fs_type(part)
     post_message("Retrieving '#{part}' partition filesystem type...")
-    @logger.debug "get #{part} partition filesystel type"
+    @logger.debug "get #{part} partition filesystem type"
     # get partition device and then its fs type
     part_fs_type = remote_handler().get_partition_fs_type(part)
     @logger.debug "Found '#{part_fs_type}' as filesystem type"
@@ -696,7 +696,7 @@ module StateTransitionHelper
   # Get partition filesytem type and label
   def get_partition_fs_type_and_label(part)
     post_message("Retrieving '#{part}' partition filesystem type...")
-    @logger.debug "get #{part} partition filesystel type"
+    @logger.debug "get #{part} partition filesystem type"
     # get partition device and then its fs type
     part_fs_type = remote_handler().get_partition_fs_type(part)
     @logger.debug "Found '#{part_fs_type}' as filesystem type"

data/lib/help/v_cloud_api_handler.rb

@@ -0,0 +1,155 @@
+require "base64"
+require "xmlsimple"
+
+class VCloudApiHandler
+  def initialize(username, password, url_endpoint, logger = Logger.new(STDOUT))
+    @username = username
+    @password = password
+    @url_endpoint = url_endpoint
+    @logger = logger
+  end
+
+  def versions
+    res = Net::HTTP.get "#{@url_endpoint}", '/api/versions'
+    return XmlSimple.xml_in(res)
+  end
+
+  def login
+    url = URI.parse("https://services.vcloudexpress.terremark.com/api/v0.8a-ext1.6/login")    
+    req = Net::HTTP::Post.new(url.path)
+    req.basic_auth("#{@username}", "#{@password}")
+    req.add_field('Content-Length','0')
+    req.add_field('Content-Type', 'application/vdn.vmware.vCloud.orgList+xml')
+    @logger.debug "--- Request-Header:"
+    req.each_header do |key, name|
+      @logger.debug "#{key}: #{name}"
+    end
+    res = http_request(url, req)
+    @vcloud_token = res.get_fields("Set-Cookie")
+    xmlbody = XmlSimple.xml_in(res.body)
+    @organization_link = xmlbody["Org"].first["href"]
+    return xmlbody
+  end
+
+  def org
+    res = generic_get_request(@organization_link)
+    xmlbody = XmlSimple.xml_in(res.body)
+    xmlbody["Link"].each() {|info|
+      @logger.info "org: found info on #{info.inspect}"
+      case info['type']
+        when "application/vnd.vmware.vcloud.vdc+xml"
+          @vdc_link = info['href']
+        when "application/vnd.vmware.vcloud.catalog+xml"
+          @catalog_link = info['href']
+        when "application/vnd.vmware.vcloud.tasksList+xml"
+          @taskslist_link = info['href']
+        when "application/vnd.tmrk.vcloudExpress.keysList+xml"
+          @keyslist_link = info['href']
+        else
+          @logger.info "could not identify #{info['type']} for org"
+      end
+    }
+    return xmlbody
+  end
+
+  def vdc
+    @server_links = []
+    @network_links = []    
+    res = generic_get_request(@vdc_link)
+    xmlbody = XmlSimple.xml_in(res.body)    
+    xmlbody['ResourceEntities'].first['ResourceEntity'].each() do |info|
+      @logger.info "vdc: found info on #{info.inspect}"
+      case info['type']      
+        when "application/vnd.vmware.vcloud.vApp+xml"
+          @server_links << info['href']
+      else
+        @logger.info "could not identify #{info['type']} for vdc"
+      end
+    end
+    @logger.debug "@server_links = #{@server_links.inspect}"
+    xmlbody['AvailableNetworks'].first['Network'].each() do |info|
+      @logger.debug "vdc: found info on #{info.inspect}"
+      case info['type']
+        when "application/vnd.vmware.vcloud.network+xml"
+          @network_links << info['href']
+      else
+        @logger.info "could not identify #{info['type']} for vdc"
+      end
+    end
+    @logger.debug "@network_links = #{@network_links.inspect}"
+  end
+
+  def v_apps
+    @server_links.each() {|server_link|
+      generic_get_request(server_link)
+    }
+  end
+
+  def networks
+    @network_links.each() {|network_link|
+      generic_get_request(network_link)
+    }
+  end
+
+  def internet_services
+    @internet_services_link = "#{@vdc_link}/internetServices"
+    @internet_services_link.gsub!("api/v0.8a-ext1.6","api/extensions/v1.6")
+    res = generic_get_request(@internet_services_link)
+    xmlbody = XmlSimple.xml_in(res.body)
+    return xmlbody
+  end
+
+  def server_ip_address(server_ip)
+    server_link = "https://services.vcloudexpress.terremark.com/api/v0.8a-ext1.6/vapp/#{server_ip}"
+    res = generic_get_request(server_link)
+    xmlbody = XmlSimple.xml_in(res.body)
+    ip_address = xmlbody["NetworkConnectionSection"].first["NetworkConnection"].first["IpAddress"]
+    @logger.debug "Ip: #{ip_address}"
+    #"NetworkConnectionSection"=>[{"NetworkConnection"=>[{"IpAddress"=>["10.114.117.11"],
+    ip_address
+  end
+
+  def firewall_rules
+    
+  end
+
+  #private
+
+  def generic_get_request(full_url)
+    raise Exception.new("no url") if full_url == nil
+    @logger.debug "generic request: url = #{full_url.inspect}"
+    url = URI.parse(full_url)
+    req = Net::HTTP::Get.new(url.path)
+    prepare_request(req)
+    return http_request(url, req)
+  end
+
+  def prepare_request(request)
+    raise Exception.new("no vcloud-token") if @vcloud_token == nil
+    request.add_field('Content-Length','0')
+    request.add_field('Content-Type', 'application/vdn.vmware.vCloud.orgList+xml')
+    request.add_field('Cookie', @vcloud_token)
+    
+    @logger.debug "--- Request-Header:"
+    request.each_header do |key, name|
+      @logger.debug "#{key}: #{name}"
+    end
+  end
+
+  def http_request(url, request)
+    http_session = Net::HTTP.new(url.host, url.port)
+    http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    http_session.use_ssl = true
+    res = http_session.start {|http|
+      http.request(request)
+    }
+
+    @logger.debug "--- Response-Header:"
+    res.each_header do |key, name|
+      @logger.debug "#{key}: #{name}"
+    end
+    xmlbody = XmlSimple.xml_in(res.body)
+    @logger.debug "response-body: #{xmlbody.inspect}"
+    return res
+  end
+end

data/lib/help/v_cloud_transition_helper.rb

@@ -0,0 +1,58 @@
+require 'net/scp'
+
+# Contains methods that are used by the scripts in the state-machines. Since
+# they are reused by different scripts, they are factored into this module
+module VCloudTransitionHelper
+
+  def retrieve_ip_services
+    @context[:vcloud_internet_services] = []
+    vcloud_api_handler.login
+    vcloud_api_handler.org
+    vcloud_api_handler.vdc
+    res = vcloud_api_handler.internet_services
+    puts "retrieved: #{res.inspect}"
+    res['InternetService'].each() {|is|
+      port = is['Port'].first.to_i
+      ip = is['PublicIpAddress'].first['Name'].first #TODO: several IPs may be defined here?
+      id = is['PublicIpAddress'].first['Id'].first
+      @context[:vcloud_internet_services] << {:port => port, :ip => ip, :id => id}
+    }
+  end
+
+  protected
+
+  #setting/retrieving handlers
+
+  def remote_handler()
+    if @remote_handler == nil
+      if @context[:remote_command_handler] == nil
+        @context[:remote_command_handler] = RemoteCommandHandler.new
+      else
+        @remote_handler = @context[:remote_command_handler]
+      end
+    end
+    @remote_handler
+  end
+
+  def remote_handler=(remote_handler)
+    @remote_handler = remote_handler
+  end
+
+  def vcloud_api_handler()
+    if @vcloud_api_handler == nil
+      @vcloud_api_handler = @context[:vcloud_api_handler]
+    end
+    @vcloud_api_handler
+  end
+
+  def vcloud_api_handler=(vcloud_api_handler)
+    @vcloud_api_handler = vcloud_api_handler
+  end
+
+  def post_message(msg)
+    if @context[:script] != nil
+      @context[:script].post_message(msg)
+    end
+  end
+
+end

data/lib/scripts/ec2/ami2_ebs_conversion.rb

@@ -34,6 +34,9 @@ class Ami2EbsConversion < Ec2Script
     if @input_params[:security_group_name] == nil
       @input_params[:security_group_name] = "default"
     end
+    if @input_params[:ami_id] == nil && !(@input_params[:ami_id] =~ /^ami-.*$/)
+      raise Exception.new("Invalid AMI ID specified: #{@input_params[:ami_id]}")
+    end
     ec2_helper = Ec2Helper.new(@input_params[:ec2_api_handler])
     if !ec2_helper.check_open_port(@input_params[:security_group_name], 22)
       raise Exception.new("Port 22 must be opened for security group #{@input_params[:security_group_name]} to connect via SSH")

data/lib/scripts/ec2/audit_via_ssh.rb

@@ -4,7 +4,6 @@ require "help/remote_command_handler"
 require "help/ec2_helper"
 require "audit/lib/audit"
 require "AWS"
-require 'pp'
 
 # Audit an AMI or an instance via an SSH connection using a specific benchmark
 #
@@ -42,6 +41,7 @@ class AuditViaSsh < Ec2Script
     else
       raise Exception.new("Invalid Audit '#{@input_params[:audit_type]}' specified")
     end
+
     ec2_helper = Ec2Helper.new(@input_params[:ec2_api_handler])
     if !ec2_helper.check_open_port(@input_params[:sec_grp_name], 22)
       raise Exception.new("Port 22 must be opened for security group 'default' to connect via SSH")
@@ -81,13 +81,21 @@ class AuditViaSsh < Ec2Script
       @context[:instance_id] = instance_infos[0]
       @context[:public_dns_name] = instance_infos[1]
       @context[:tmp_dir] = tmp_dir
-      #puts "DEBUG: Audit Scripts"
-      #pp @context
 
       Dir::mkdir(tmp_dir)
       if FileTest::directory?(tmp_dir)
         post_message("local temporary directory created")
       end
+ 
+      if @context[:audit] == nil
+        @context[:audit] = Audit.new(:benchmark => @context[:benchmark_file], :attachment_dir => @context[:tmp_dir],
+                                     :connection_type => :ssh, 
+                                     :connection_params => {:user => @context[:ssh_user],
+                                                            :keys => @context[:ssh_key_file],
+                                                            :host => @context[:public_dns_name],
+                                                            :paranoid => false}, 
+                                     :logger => nil)
+      end
 
       LaunchAuditViaSsh.new(@context)
     end
@@ -96,20 +104,21 @@ class AuditViaSsh < Ec2Script
   # Launch the audit via SSH
   class LaunchAuditViaSsh < AuditViaSshState
     def enter
-      audit = Audit.new(:benchmark => @context[:benchmark_file], :attachment_dir => @context[:tmp_dir],
-                        :connection_type => :ssh, 
-                        :connection_params => {:user => @context[:ssh_user],
-                                               :keys => @context[:ssh_key_file],
-                                               :host => @context[:public_dns_name],
-                                               :paranoid => false}, 
+      audit = @context[:audit]
+      #audit = Audit.new(:benchmark => @context[:benchmark_file], :attachment_dir => @context[:tmp_dir],
+      #                  :connection_type => :ssh, 
+      #                  :connection_params => {:user => @context[:ssh_user],
+      #                                         :keys => @context[:ssh_key_file],
+      #                                         :host => @context[:public_dns_name],
+      #                                         :paranoid => false}, 
                                                #:timeout => 120,
                                                #:verbose => :warn},
-                                               :logger => nil)
+      #                                         :logger => nil)
       audit.start(false)
       @context[:result][:audit_test] = []
       audit.results.each() {|key, value|
         if key =~ /^SSH_.*$/ || key =~ /^APACHE2_.*$/
-          #puts "DEBUG: Key: #{key}, Result: #{value.result}, Desc: #{value.rule.description}"
+          puts "DEBUG: Key: #{key}, Result: #{value.result}, Desc: #{value.rule.description}"
           @context[:result][:audit_test] << {:name => key, :desc => value.rule.description, :status => value.result}
           post_message("== > Test #{key}: Status: #{value.result.eql?("pass") ? "OK" : "NOK"}\n  Desc: #{value.rule.description}")
         end

data/lib/scripts/ec2/copy_ami.rb

@@ -38,6 +38,9 @@ class CopyAmi < Ec2Script
   end
 
   def check_input_parameters()
+    if @input_params[:ami_id] == nil && !(@input_params[:ami_id] =~ /^ami-.*$/)
+      raise Exception.new("Invalid AMI ID specified: #{@input_params[:ami_id]}")
+    end
     ec2_helper = Ec2Helper.new(@input_params[:ec2_api_handler])
     if ec2_helper.ami_prop(@input_params[:ami_id], 'rootDeviceType') != "ebs"
       raise Exception.new("must be an EBS type image")

data/lib/scripts/ec2/download_snapshot.rb

@@ -41,6 +41,9 @@ class DownloadSnapshot < Ec2Script
     if !ec2_helper.check_open_port(@input_params[:security_group_name], 22)
       raise Exception.new("Port 22 must be opened for security group #{@input_params[:security_group_name]} to connect via SSH")
     end
+    if !ec2_helper.check_open_port(@input_params[:security_group_name], 80)
+      raise Exception.new("Port 80 must be opened for security group #{@input_params[:security_group_name]} to make the download link work")
+    end
     if @input_params[:source_device] == nil
       @input_params[:source_device] = "/dev/sdj1"
     end

data/lib/scripts/ec2/ec2_script.rb

@@ -21,7 +21,7 @@ class Ec2Script
     @input_params[:result] = @result
     @input_params.each() do |param_key, param_value|
       if [:logger, :ec2_api_handler, :target_ec2_handler, :remote_command_handler,
-        :source_ssh_keydata, :target_ssh_keydata
+        :source_ssh_keydata, :target_ssh_keydata, :ssh_keydata
         ].include?(param_key.to_s.to_sym)
         @logger.debug("INPUT PARAM #{param_key} is set [but not logged]")
       else

data/lib/scripts/vCloud/open_port_checker_vm.rb

@@ -0,0 +1,82 @@
+require "help/script_execution_state"
+require "help/remote_command_handler"
+require "scripts/vcloud/v_cloud_script"
+
+# Identifies all open internet services and checks if
+# there are actually used by a service. Note: this script depends
+# on a propriatary version of the vCloud API implemented by Terremark.
+#
+
+class OpenPortCheckerVm < VCloudScript
+  # Input parameters
+  # * vcloud_api_handler => object that allows to access the vCloud service
+  def initialize(input_params)
+    super(input_params)
+  end
+
+  def check_input_parameters()
+    if @input_params[:vcloud_api_handler] == nil
+      raise Exception.new("no vCloud handler specified")
+    end
+  end
+
+  def load_initial_state()
+    OpenPortCheckerState.load_state(@input_params)
+  end
+
+  private
+
+  # Here begins the state machine implementation
+  class OpenPortCheckerState < ScriptExecutionState
+    def self.load_state(context)
+      state = context[:initial_state] == nil ? RetrievingInternetServices.new(context) : context[:initial_state]
+      state
+    end
+
+  end
+
+  # Nothing done yet. Retrieve all internet services.
+  class RetrievingInternetServices < OpenPortCheckerState
+    def enter
+      retrieve_ip_services()
+      CheckingInternetServices.new(@context)
+    end
+  end
+
+  # Got all internet services. Go through them and check the ports.
+  class CheckingInternetServices < OpenPortCheckerState
+    def enter
+      @context[:result][:port_checks] = []
+      @context[:vcloud_internet_services].each() do |is|
+        port = is[:port]
+        ip = is[:ip]
+        identifier = is[:id]
+        begin
+          result = @context[:remote_command_handler].is_port_open?(ip, port)
+          post_message("check port #{port} on IP #{ip} => #{result ? "successful" : "failed"}")
+        rescue Exception => e
+          @logger.warn("exception during executing port check: #{e}")
+        end
+        @context[:result][:port_checks] << {:ip => ip, :id => identifier,
+          :port => port, :success => result
+        }
+      end
+      AnalysisDone.new(@context)
+    end
+  end
+
+  # Nothing done yet. Retrieve all security groups
+  class AnalysisDone < OpenPortCheckerState
+    def enter
+      Done.new(@context)
+    end
+  end
+
+  # Script done.
+  class Done < OpenPortCheckerState
+    def done?
+      true
+    end
+  end
+
+end

data/lib/scripts/vCloud/v_cloud_script.rb

@@ -0,0 +1,110 @@
+# Base class for any script for vCloud.
+class VCloudScript
+  # Initialization. Common Input parameters:
+  # * vcloud_api_handler => used to connect to the API (needs user/password/api-url)
+  # * logger => allows to pass a ruby logger object used for logging (optional, default is a stdout logger with level WARN)
+  # Scripts may add specific key/value pairs.
+  def initialize(input_params)
+    @input_params = input_params
+    @state_change_listeners = []
+    @progress_message_listeners = []
+    if input_params[:logger] == nil
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::WARN
+      input_params[:logger] = @logger
+    else
+      @logger = input_params[:logger]
+    end
+    @result = {:done => false, :failed => false}
+    @input_params[:result] = @result
+    @input_params.each() do |param_key, param_value|
+      if [:logger, :vcloud_api_handler, :target_ec2_handler, :remote_command_handler,
+        :source_ssh_keydata, :target_ssh_keydata, :ssh_keydata
+        ].include?(param_key.to_s.to_sym)
+        @logger.debug("INPUT PARAM #{param_key} is set [but not logged]")
+      else
+        @logger.debug("INPUT PARAM #{param_key} = #{param_value.inspect}")
+      end
+    end
+  end
+
+  def register_state_change_listener(listener)
+    @state_change_listeners << listener
+  end
+
+  def register_progress_message_listener(listener)
+    @progress_message_listeners << listener
+  end
+
+  # Check input parameters (in @input_parameters object variable)
+  # and set default values.
+  # Abstract method to be implemented by extending classes.
+  def check_input_parameters()
+    raise Exception.new("check_input_parameters must be implemented")
+  end
+
+  # Load the initial state for the script.
+  # Abstract method to be implemented by extending classes.
+  def load_initial_state()
+    raise Exception.new("load_initial_state must be implemented")
+  end
+
+  # Executes the script.
+  def start_script()
+    # optional parameters and initialization
+    check_input_parameters()
+    @input_params[:script] = self
+    begin
+      current_state = load_initial_state()
+      @state_change_listeners.each() {|listener|
+        current_state.register_state_change_listener(listener)
+      }
+      end_state = current_state.start_state_machine()
+      if end_state.failed?
+        @result[:failed] = true
+        @result[:failure_reason] = current_state.end_state.failure_reason
+        @result[:end_state] = current_state.end_state
+      else
+        @result[:failed] = false
+      end
+    rescue Exception => e
+      @logger.warn "exception during execution: #{e}"
+      @logger.warn e.backtrace.join("\n")
+      err = e.to_s
+      err += " (in #{current_state.end_state.to_s})" unless current_state == nil
+      @result[:failed] = true
+      @result[:failure_reason] = err
+      @result[:end_state] = current_state.end_state unless current_state == nil
+    ensure
+      begin
+      @input_params[:remote_command_handler].disconnect
+      rescue Exception => e2
+      end
+    end
+    #
+    @result[:done] = true
+  end
+
+
+  # Return a hash of results. Common values are:
+  # * :done => is true when the script has terminated, otherwise false
+  # * :failed => is false when the script succeeded
+  # * :failure_reason => returns a failure reason (string)
+  # * :end_state => returns the state, in which the script terminated (#Help::ScriptExecutionState)
+  # Scripts may add specific key/value pairs.
+  # * 
+  # Returns a hash with the following information:
+  # :done => if execution is done
+  #
+  def get_execution_result
+    @result
+  end
+
+  def post_message(message, level = Logger::DEBUG)
+    @progress_message_listeners.each() {|listener|
+      listener.new_message(message, level)
+    }
+  end
+  
+end
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: CloudyScripts
 version: !ruby/object:Gem::Version 
-  hash: 103
+  hash: 121
   prerelease: false
   segments: 
-  - 1
-  - 10
-  - 44
-  version: 1.10.44
+  - 2
+  - 11
+  - 45
+  version: 2.11.45
 platform: ruby
 authors: 
 - Matthias Jung
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-07 00:00:00 +02:00
+date: 2011-09-16 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -257,6 +257,8 @@ files:
 - lib/help/script_execution_state.rb
 - lib/help/state_change_listener.rb
 - lib/help/state_transition_helper.rb
+- lib/help/v_cloud_api_handler.rb
+- lib/help/v_cloud_transition_helper.rb
 - lib/scripts/ec2/ami2_ebs_conversion.rb
 - lib/scripts/ec2/audit_via_ssh.rb
 - lib/scripts/ec2/copy_ami.rb
@@ -268,6 +270,8 @@ files:
 - lib/scripts/ec2/open_port_checker.rb
 - lib/scripts/ec2/port_range_detector.rb
 - lib/scripts/ec2/snapshot_optimization.rb
+- lib/scripts/vCloud/open_port_checker_vm.rb
+- lib/scripts/vCloud/v_cloud_script.rb
 has_rdoc: true
 homepage: http://elastic-security.com
 licenses: []