kiel 0.0

data/lib/kiel.rb

@@ -0,0 +1,249 @@
+require 'rake'
+require 'kiel/scm/git'
+require 'kiel/cloud/aws'
+
+# Kiel tries to make the task to create cloud images easier by braking the whole installation into smaller, reproducible tasks.
+# Each step is versioned by a version control system like git or subversion. Each installation step is described by a file
+# containing a capistrano script. Kiel assumes that there is a specific order in which the steps have to be executed. 
+#
+# The purpose of splitting the installation of a machine image into smaller tasks is to save time when debugging the 
+# installation and save time, when little changes have to be made to the installation. 
+#  
+# If one step fails, all subsequent installation steps might fail too. But when one step succeeds, that step can be
+# used as base for all subsequence steps. 
+#
+# License::   Distributes under the MIT license
+
+module Kiel
+    #--
+    RECOGNIZED_STEP_OPTIONS = [ :name, :task, :scm_name, :setup_name, :description ]
+    DEFAULT_OPTIONS = {}
+    RECOGNIZED_OPTIONS = [ :scm, :cloud, :setup, :base_image, :root_dir ]
+     
+    class Implementation
+        def initialize defaults
+            @defaults = defaults.dup
+        end
+        
+        # this getters defer the construction of expensive devices to the latest moment possible
+        def scm
+            @defaults[ :scm ] ||= SCM::Git.new
+        end
+        
+        def cloud
+            @defaults[ :cloud ] ||= Cloud::AWS.new
+        end
+        
+        def setup
+            @defaults[ :setup ] ||= Setup::Capistrano.new
+        end 
+    
+        def expand_path file_name
+            @defaults[ :root_dir ] ||= Dir.pwd
+            
+            if file_name.kind_of? Array 
+                file_name.collect { |f| File.expand_path f, @defaults[ :root_dir ] }
+            else                
+                File.expand_path file_name, @defaults[ :root_dir ]
+            end
+        end
+        
+        def build_tags step, steps
+            steps.inject( { 'image_type' => step[ :name ].to_s, step[ :name ].to_s => step[ :version ].to_s } ) do | t, s |
+                 t.merge s[ :name ].to_s => s[ :version ].to_s
+            end
+        end
+        
+        def initial_image_id step, base_steps
+            if base_steps.empty?
+                unless @defaults.key? :base_image
+                    raise ArgumentError, "no :base_image given. Need to know what the base image of the very first image to produce should be"
+                end     
+                { :id => @defaults[ :base_image ] }
+            else         
+                step, *steps = *base_steps        
+                { :tags => build_tags( step, steps ) }
+            end     
+        end
+    
+        def create_task step, steps
+            task = Rake::Task::define_task( step[ :task ] => steps.collect{ | s | s[ :task ] } ) do | task, arguments |
+                tags = build_tags step, steps
+
+                if cloud.exists? tags
+                    puts "image \'#{step[ :name ]}\' already up to date and exists:"
+                    tags.each{ | key, value | puts "\'#{key}\' => \'#{value}\'" }                    
+                else
+                    puts "starting instance for: \'#{step[ :name ]}\'"
+                    instance = cloud.start_instance initial_image_id( step, steps )
+                    puts "instance for: \'#{step[ :name ]}\' started."
+
+                    begin
+                        dns_name    = cloud.dns_name instance
+                        expand_step = step.dup.merge( setup_name: expand_path( step[ :setup_name ] ) )
+
+                        puts "excuting installation for: \'#{step[ :name ]}\'"
+                        setup.execute expand_step, dns_name  
+                        puts "installation for: \'#{step[ :name ]}\' done."
+
+                        puts "storing image for: \'#{step[ :name ]}\'"
+                        cloud.store_image instance, tags
+                        puts "image for: \'#{step[ :name ]}\' stored"
+                    rescue
+                        cloud.stop_instance instance
+                        raise
+                    end                     
+                end                     
+            end
+
+            task.add_description( step[ :description ] ) if step.key? :description
+            task
+        end
+        
+        def add_versions steps
+            steps.collect() do | step | 
+                name = step[ :scm_name ] == '*' ? '*' : expand_path( step[ :scm_name ] )
+                step.merge( version: scm.version( name ) ) 
+            end
+        end
+
+        private :setup, :cloud, :scm, :initial_image_id, :build_tags
+    end
+    
+    # checks that all keys in options are valid
+    def self.check_options options 
+        options.each_key{ | key |
+            raise ArgumentError, "Unrecognized option: \'#{key}\'" unless RECOGNIZED_OPTIONS.include? key
+        }
+    end
+    
+    def self.expand_steps steps
+        raise ArgumentError, "no steps given" if steps.empty?
+        
+        # convert all steps to hashes and check the given parameters
+        steps = steps.collect do | s; step |
+            if s.respond_to? :to_hash
+                step = s.to_hash.dup
+                step.each { | key, value | 
+                    raise ArgumentError, "unrecognized step option: \'#{key}\'" unless RECOGNIZED_STEP_OPTIONS.include? key
+                }
+
+                raise ArgumentError, "every step have to have at least a name" unless step.key? :name 
+
+                step
+            elsif s.respond_to? :to_s
+                { :name => s.to_s }
+            else
+                raise ArgumentError, "a step have to be a string, symbol or a hash"
+            end
+        end
+
+        steps.first[ :scm_name ] = '*' unless steps.first.key? :scm_name  
+
+        # merge in defaults
+        steps = steps.collect do | step |
+            { 
+                :task => step[ :name ], 
+                :scm_name => "#{step[ :name ]}.rb"
+            }.merge step
+        end.collect do | step |
+            {
+                :setup_name => step[ :scm_name ] == '*' ? "#{step[ :name ]}.rb" : step[ :scm_name  ] 
+            }.merge step
+        end
+        
+        steps
+    end
+
+    # defines the +steps+ necessary to build an image by constructing Rake::Tasks that depend on each other. The 
+    # dependencies are defined in the order the steps are given. Every step depends on all other steps following
+    # that step in the list of given +steps+.
+    #
+    # Each step produces a new machine image, by starting a server in the cloud with the previous image,
+    # adding a defined set of installation instructions and than saving the resulting image for the next step.
+    # 
+    # Every step is defined by hash with the following keys:
+    #
+    # :name:: The name of the step. The name is used to name a tag in the resulting image. The value of the tag is 
+    #         the source code version of the sources of that step. By default +name+ is expanded to +name.rb+ in the
+    #         current directory.
+    #
+    # :task:: The name of the Rake::Task to be created for that step. If not given, the +name+ is used.
+    #
+    # :scm_name:: The name that is passed to the source code management to determine the version of the description
+    #             of the step. If not given, +name+ is expanded to +name.rb+ in the current directory. For the first
+    #             element this defaults to '*', which is a special notation for the latest version of the repository.
+    # 
+    # :setup_name:: The name of the script to be executed. This defaults to :scm_name if given and not '*' or to 
+    #               :name + '.rb' 
+    #
+    # :description:: Optional description for the task to be created.
+    #
+    # A step can be given by just one string or symbol, both following lines will result in the same images created.
+    #    Kiel::image [ :stepA, :stepB ] 
+    #    Kiel::image [ { :name => 'stepA', :task => 'stepA', :scm_name => '*', :setup_name ='stepA.rb' }, 
+    #                  { :name => 'stepB', :task => 'stepB', :scm_name => 'stepB.rb' } ]
+    #
+    # +options+ is a set of configurations that can be used to override global options set by Kiel::set_defaults. 
+    # Possible options are:
+    #
+    # :scm:: An instance of the +source code management+ used to retrieve version informations. By default this will
+    #        be an instance of +Kiel::SCM::Git+.
+    # 
+    # :setup:: An instance of the device used to execute steps to execute the installations steps. By default this will
+    #          be an instance of +Kiel::Setup::Capistrano+.
+    # 
+    # :cloud:: An instance of the cloud provider to lookup images and to access cloud instances. By default this will
+    #          be an instance of +Kiel::Cloud::AWS+
+    # 
+    # :base_image:: A cloud image id that is used as base for the very first step. This is the right most argument in 
+    #               the list of +steps+.
+    # 
+    # :root_dir:: Root directory, where all file names are bassed on. If the options is not given, the current directory is used
+    #
+    # Example:
+    #    Kiel::image [ 'application', 'base' ], setup: Kiel::Setup::Capistrano.new, base_image: 'ami-6d555119'
+    #
+    # Will assume that every new version in the repository should lead to a new image based on an base image. The
+    # layout of the base image is defined by base.rb and the base images is only recreated when the version of base.rb
+    # changes. The base image is build by starting a cloud image with the id 'ami-6d555119'. To setup the base-image,
+    # base.rb is executed by a newly created Kiel::Setup::Capistrano instance. The resulting base image will be stored
+    # with the tags: 
+    #    { 'image_type' => 'base', 'base' => '<version of base.rb>' }.
+    #
+    # An application image is build by starting a cloud server with the base image and executing the steps provided by
+    # application.rb. The application image is then stored with the following tags:
+    #    { 'iamge_type' => 'application', 'application' => '<version of the overall repository>, 'base' => '<version of base.rb>' }. 
+    def self.image steps, options = {} 
+        check_options( options ) 
+        
+        implemenation = Implementation.new defaults().merge( options )
+        steps = expand_steps steps
+        steps = implemenation.add_versions steps
+        
+        while !steps.empty? do
+            step, *steps = *steps
+
+            implemenation.create_task step.dup, steps.dup
+        end
+    end
+    
+    private_class_method :expand_steps, :check_options
+
+    # set the global defaults that are applied to Kiel::image 
+    def self.set_defaults defs
+        check_options defs
+        @@defaults ||= DEFAULT_OPTIONS.dup
+        @@defaults.merge! defs
+    end
+     
+    def self.reset_defaults
+        @@defaults = nil
+    end
+            
+    # returns the global defaults that are applied to every call to Kiel::image
+    def self.defaults
+        @@defaults ||= DEFAULT_OPTIONS.dup
+        @@defaults
+    end 
+end

data/lib/kiel/cloud/aws.rb

@@ -0,0 +1,163 @@
+require 'digest/sha1'
+
+module Kiel
+    module Cloud
+        INSTANCE_STARTUP_TIMEOUT    = 120
+        RECOGNIZED_OPTIONS = [ :region, :credentials, :instance, :start_options ]
+        
+        # Implements the connection to the Amazon Web Services (AWS). The current implementation works for one 
+        # configured region. The default server is a small EC2 instance.
+        class AWS
+            # The contructor takes the following configuration options:
+            #
+            # :region:: A string naming the region to be used. If no region is given, the default region is used.
+            #
+            # :credentials:: A hash containing the fields 'access_key_id' and 'secret_access_key' with the credential
+            #                information to your amazon account.
+            #
+            # :instance:: An instance of the AWS::EC2. If that instance is given, no +credentials:+ should be given.
+            #             Kiel::Cloud::AWS will instead use this instance.     
+            #
+            # :start_options:: Options that are applied to EC2::InstanceCollection.create (siehe aws-sdk for more
+            #                  details). The aws_tests.rb uses the :key_name and :security_groups options to set the
+            #                  name of the used ssh key and a security group, where ssh is enabled.
+            def initialize options = {}
+                require 'aws/ec2'
+
+                options.each_key do | key |
+                    raise ArgumentError, "unrecognized option \'#{key}\'" unless RECOGNIZED_OPTIONS.include? key
+                end
+                                    
+                @ec2 = options[ :instance ]
+                @start_options = options[ :start_options ] || {}
+                
+                if @ec2 
+                    puts "\'credentials\' ignored as an instance was given too" if options.key? :credentials
+                else
+                    ::AWS.config( options[ :credentials ]  )
+    
+                    @ec2 = ::AWS::EC2.new
+                    @ec2 = @ec2.regions[ options[ :region ] ] if options.key? :region
+                end
+            end
+
+            def all_images_by_tags tags
+                images = @ec2.images.with_owner('self').tagged( tags.first.first )
+    
+                images = images.select do | image |
+                    image_tags = image.tags.to_h
+                    image_tags.merge( tags ) == image_tags 
+                end
+                
+                images
+            end    
+
+            def image_by_tags tags
+                images = all_images_by_tags tags
+                
+                raise "#{images.size} are tagged with the given tags: #{tags.inspect}" if images.size > 1 
+                images.size == 1 ? images.first : nil 
+            end
+            
+            def wait_for_ec2 instance
+                puts "waiting for EC2 instance to start."
+                sleep_count = INSTANCE_STARTUP_TIMEOUT
+                while instance.status == :pending and sleep_count != 0 do 
+                    sleep 1
+                    sleep_count = sleep_count - 1
+                end
+            end
+
+            def wait_for_image image
+                image_state = :pending                              
+                while image_state == :pending do
+                    begin 
+                        image_state = image.state
+                    rescue => e
+                        puts "err: #{e.inspect}"
+                    end     
+                       
+                    sleep 1
+                    STDOUT.write '.'
+                end
+                puts ''
+            end
+
+            private :image_by_tags, :wait_for_ec2, :wait_for_image
+            
+            # starts a server instance in the cloud, returning a handle to that instance.
+            # the image is either named by an image id +:id => 'image_id'+ or by a set of tags that match for
+            # just one image +:tags => { 'image_type' => 'application', 'base' => '34' }+
+            def start_instance image_name
+                options = @start_options.merge( if image_name.key?( :id ) 
+                    { image_id: image_name[ :id ] }
+                else
+                    tags  = image_name[ :tags ]
+                    image = image_by_tags tags
+                    raise RuntimeError, "no image with tags: \'#{tags}\' found to start an instance" unless image
+                    
+                    { image_id: image.id }
+                end )
+                
+                instance = @ec2.instances.create( options )
+                
+                begin
+                    wait_for_ec2 instance
+                    puts "ec2 instance \'#{instance.dns_name}\' started."
+                rescue 
+                    instance.terminate
+                    raise
+                end 
+                
+                instance
+            end
+            
+            # store the given +instance+ and add the hash of +tags+ to the image. 
+            def store_image instance, tags
+                begin
+                
+                    puts "waiting 2 minutes before starting to take the image..."
+                    sleep 120
+                    puts "creating images..."
+    
+                    image = @ec2.images.create( 
+                        :instance_id => instance.id,
+                        :no_reboot => true,
+                        :description => "automaticaly created #{tags[ 'image_type' ]} image",
+                        :name => "#{tags[ 'image_type' ]} #{Digest::SHA1.hexdigest tags.inspect}" )
+                       
+                    wait_for_image image
+    
+                    tags.each do | key, value |
+                        image.add_tag( key, :value => value )
+                    end               
+                ensure
+                    stop_instance instance
+                end
+            end
+            
+            # stops the given instance.
+            def stop_instance instance
+                begin
+                    instance.terminate
+                rescue
+                end                    
+            end
+
+            # returns true, if an image with the given tags exists
+            def exists? tags
+                raise ArgumentError, "AWS.exists? with empty tags" if tags.empty?
+                
+                image_by_tags tags 
+            end
+            
+            def dns_name instance
+                instance.dns_name
+            end
+            
+            # deletes the given image
+            def delete_image image_name
+            end
+        end
+    end
+end

data/lib/kiel/cloud/mock.rb

@@ -0,0 +1,76 @@
+
+module Kiel
+    module Cloud
+    
+        # Implementation of the Cloud access-Interface
+        # The implementation assumes that a cloud provider provides machine images, used to start machines,
+        # that this images can a not unique set of tags and a unique id. Where the id is provided by the cloud provider 
+        # tags are provided by the user. Kiel used the tags to store version informations to an image and uses this 
+        # set of tags to identify an image.
+        #
+        # The implementation assumes that a cloud provider allows to start a server with a machine image as parameter
+        # and that the resulting instance has a public dns_name to be reachable.
+        class Mock
+            # +existing_images+ simulates an initial set of existing machine images, +dns_names+ provides a set of 
+            # names that are assigned to newly created cloud instances.
+            def initialize existing_images = [], dns_names = [] 
+                @names = dns_names.dup
+                @calls = [] 
+                @images = [ existing_images.dup ].flatten
+                @running = []
+                @next_instance = 0
+            end
+            
+            # starts a server instance in the cloud, returning a handle to that instance.
+            # the image is either named by an image id +:id => 'image_id'+ or by a set of tags that match for
+            # just one image +:tags => { 'image_type' => 'application', 'base' => '34' }+
+            def start_instance image_name
+                raise ArgumentError, "image_name must contain exactly one identification" unless image_name.size == 1
+
+                @running << @next_instance
+                @next_instance += 1
+                @running.last
+            end
+            
+            # store the given +instance+ under the given +image_name+ and add the hash of +tags+ to the image. 
+            def store_image instance, tags
+                image = { id: instance, tags: tags }
+                @calls << { func: :store_image, args: image }
+                @images << image
+                stop_instance instance
+            end
+            
+            # stops the given instance.
+            def stop_instance instance
+                unless @running.delete( instance ) 
+                    raise RuntimeError, "there is no instance #{instance} running"
+                end                    
+            end
+
+            # returns true, if an image with the given tags exists
+            def exists? tags
+                @images.detect { | image | image[ :tags ] == tags }
+            end
+
+            # returns the dns name from an instance
+            def dns_name instance
+                "#{instance}"
+            end
+            
+            #--
+            def calls
+                @calls
+            end 
+            
+            #--
+            def running_instances
+                @running
+            end
+            
+            #--
+            def stored_images
+                @images 
+            end
+        end
+    end
+end

data/lib/kiel/scm/git.rb

@@ -0,0 +1,21 @@
+require 'digest/sha1'
+
+module Kiel
+    module SCM
+        class Git
+            def single_version file
+                result = `git rev-list --max-count 1 HEAD #{file}`
+                result.gsub( /\n$/, '' )
+            end
+            
+            private :single_version
+            
+            def version file
+                files = [ file == '*' ? '' : file ].flatten
+                return single_version( files.first ) if files.size == 1 
+
+                files.sort.inject( '' ) { | sum, file | Digest::SHA1.hexdigest sum + single_version(file) }                
+            end
+        end
+    end
+end

data/lib/kiel/scm/mock.rb

@@ -0,0 +1,24 @@
+
+module Kiel
+    module SCM
+    
+        # A mock, implementing the interface to a source code managment system
+        class Mock
+            def initialize versions = {}
+                @versions = versions
+            end
+                            
+            # returns the latest version of the given file 
+            def version step
+                step = step.to_s
+                raise RuntimeError, "no mocked version for step \'#{step}\'" unless @versions.key? step
+                @versions[ step ]
+            end
+            
+            #--
+            def versions hash
+                @versions = hash
+            end
+        end
+    end
+end

data/lib/kiel/setup/capistrano.rb

@@ -0,0 +1,46 @@
+require 'yaml'
+
+module Kiel
+    module Setup
+        # The Capistrano Setup component executes one task out of a given script. The name of the script is by default
+        # the name of step + '.rb' in the +:root_dir+ given to Kiel::image. The task to be executed is 'deploy:step'.
+        # The tags that will be applied to the resulting image are passed to the script as global constant ::TAGS 
+        class Capistrano
+
+            # +options+ are passed directly to Capistrano:
+            #   options.each { | key, value |
+            #      set key, value 
+            #   } 
+            # so every options that have to be set in every script should be set by using this options.
+            def initialize options = {}
+                @options = options 
+            end
+            
+            # step contains the whole step informations at least :setup_name contains the script to be executed, 
+            # :tags contains the tags that will be added to the images after setup, :version contains the version of the
+            # steps associated scm_name's file version. 
+            def execute step, dns_server
+                options = { script: step[ :setup_name ], tags: step[ :tags ], version: step[ :version ], 
+                    name: step[ :name ].to_s, capistrano_options: @options, capistrano_server: dns_server }
+
+                file = IO.popen( ['cap', '-f', File.expand_path( '../capistrano_executer.rb', __FILE__ ), step[ :name ].to_s ], 'r+' )
+
+                file.write YAML::dump( options )
+                file.close_write
+                
+                last_line = ''
+                begin
+                    begin
+                        text = file.readpartial 1024
+                        STDOUT.write text
+                        last_line += text
+                        last_line = last_line.split("\n").last
+                    end until text.empty?
+                rescue EOFError
+                end                     
+                
+                raise "Error while executing #{step[ :setup_name ]}" if last_line =~ /\+-\+-\+-\+ERORR\+-\+-\+-\+/
+            end
+        end
+    end
+end

data/lib/kiel/setup/capistrano_executer.rb

@@ -0,0 +1,42 @@
+require 'yaml'
+
+module Kiel
+    OPTIONS = YAML.load STDIN 
+    TAGS    = OPTIONS[ :tags ]
+end    
+
+Kiel::OPTIONS[ :capistrano_options ].each do | key, value |
+    set key, value
+end
+    
+role :server, Kiel::OPTIONS[ :capistrano_server ]
+
+load Kiel::OPTIONS[ :script ]
+
+task Kiel::OPTIONS[ :name ] do
+    begin
+        retry_count = 10
+        
+        begin 
+            deploy.step
+        rescue Capistrano::ConnectionError => ex
+            raise unless ex.message =~ /Errno::ECONNREFUSED|Errno::ETIMEDOUT/
+
+            puts "Connection failed"
+            retry_count -= 1
+            
+            if retry_count == 0
+                puts 'giving up...'
+                raise
+            end
+            
+            puts 'retrying...'
+            sleep 15
+            retry
+        end            
+    rescue Exception => ex
+        puts ex.message
+        puts ex.backtrace.join("\n")
+        puts '+-+-+-+ERORR+-+-+-+'
+    end        
+end

data/lib/kiel/setup/mock.rb

@@ -0,0 +1,14 @@
+module Kiel
+    module Setup
+        class Mock
+            def execute step, server
+                @steps ||= []
+                @steps << step[ :setup_name ]
+            end
+            
+            def executed_steps
+                @steps || []
+            end
+        end
+    end
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: kiel
+version: !ruby/object:Gem::Version
+  version: '0.0'
+  prerelease: 
+platform: ruby
+authors:
+- Torsten Robitzki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-30 00:00:00.000000000 Z
+dependencies: []
+description: Helper to build rake task to build cloud images step by step for easier
+  debugging and testing
+email: gemmaster@robitzki.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/kiel/cloud/aws.rb
+- lib/kiel/cloud/mock.rb
+- lib/kiel/scm/git.rb
+- lib/kiel/scm/mock.rb
+- lib/kiel/setup/capistrano.rb
+- lib/kiel/setup/capistrano_executer.rb
+- lib/kiel/setup/mock.rb
+- lib/kiel.rb
+homepage: https://github.com/TorstenRobitzki/Kiel
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Building cloud images step by step
+test_files: []