cloudkeeper-aws 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
 - data/.gitignore +11 -0
 - data/.gitmodules +3 -0
 - data/.rubocop.yml +48 -0
 - data/.travis.yml +22 -0
 - data/Gemfile +5 -0
 - data/LICENSE +674 -0
 - data/README.md +91 -0
 - data/Rakefile +17 -0
 - data/bin/cloudkeeper-aws +5 -0
 - data/cloudkeeper-aws.gemspec +46 -0
 - data/codecov.yml +4 -0
 - data/config/cloudkeeper-aws.yml +17 -0
 - data/lib/cloudkeeper/aws.rb +16 -0
 - data/lib/cloudkeeper/aws/backend_executor.rb +76 -0
 - data/lib/cloudkeeper/aws/cli.rb +159 -0
 - data/lib/cloudkeeper/aws/cloud.rb +173 -0
 - data/lib/cloudkeeper/aws/core_connector.rb +108 -0
 - data/lib/cloudkeeper/aws/errors.rb +11 -0
 - data/lib/cloudkeeper/aws/errors/backend.rb +14 -0
 - data/lib/cloudkeeper/aws/errors/backend/appliance_not_found_error.rb +9 -0
 - data/lib/cloudkeeper/aws/errors/backend/backend_error.rb +9 -0
 - data/lib/cloudkeeper/aws/errors/backend/image_import_error.rb +9 -0
 - data/lib/cloudkeeper/aws/errors/backend/multiple_appliances_found_error.rb +9 -0
 - data/lib/cloudkeeper/aws/errors/backend/timeout_error.rb +9 -0
 - data/lib/cloudkeeper/aws/errors/image_download_error.rb +7 -0
 - data/lib/cloudkeeper/aws/errors/invalid_configuration_error.rb +7 -0
 - data/lib/cloudkeeper/aws/errors/standard_error.rb +7 -0
 - data/lib/cloudkeeper/aws/filter_helper.rb +33 -0
 - data/lib/cloudkeeper/aws/image_downloader.rb +52 -0
 - data/lib/cloudkeeper/aws/proto_helper.rb +70 -0
 - data/lib/cloudkeeper/aws/settings.rb +18 -0
 - data/lib/cloudkeeper/aws/version.rb +5 -0
 - data/lib/cloudkeeper_grpc.rb +7 -0
 - data/lib/cloudkeeper_grpc/.gitmodules +6 -0
 - data/lib/cloudkeeper_grpc/CODE_OF_CONDUCT.md +49 -0
 - data/lib/cloudkeeper_grpc/LICENSE.txt +17 -0
 - data/lib/cloudkeeper_grpc/README.md +9 -0
 - data/lib/cloudkeeper_grpc/cloudkeeper_pb.rb +59 -0
 - data/lib/cloudkeeper_grpc/cloudkeeper_services_pb.rb +31 -0
 - data/lib/cloudkeeper_grpc/constants.rb +10 -0
 - data/lib/cloudkeeper_grpc/protos/CODE_OF_CONDUCT.md +49 -0
 - data/lib/cloudkeeper_grpc/protos/LICENSE.txt +17 -0
 - data/lib/cloudkeeper_grpc/protos/README.md +3 -0
 - data/lib/cloudkeeper_grpc/protos/cloudkeeper.proto +64 -0
 - data/lib/cloudkeeper_grpc/status-codes/CODE_OF_CONDUCT.md +49 -0
 - data/lib/cloudkeeper_grpc/status-codes/LICENSE.txt +13 -0
 - data/lib/cloudkeeper_grpc/status-codes/README.md +3 -0
 - data/lib/cloudkeeper_grpc/status-codes/status-codes.yml +12 -0
 - metadata +303 -0
 
    
        data/README.md
    ADDED
    
    | 
         @@ -0,0 +1,91 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            <p align="center">
         
     | 
| 
      
 2 
     | 
    
         
            +
              <img alt="Cloudkeeper-AWS" src="https://i.imgur.com/1L2LcZ0.png" width="400"/>
         
     | 
| 
      
 3 
     | 
    
         
            +
            </p>
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            # Cloudkeeper-AWS
         
     | 
| 
      
 6 
     | 
    
         
            +
            AWS backend for [Cloudkeeper](https://github.com/the-cloudkeeper-project/cloudkeeper)
         
     | 
| 
      
 7 
     | 
    
         
            +
             
     | 
| 
      
 8 
     | 
    
         
            +
            ## What does Cloudkeeper-AWS do?
         
     | 
| 
      
 9 
     | 
    
         
            +
            Cloudkeeper-AWS is able to manage [AWS](https://aws.amazon.com/) cloud - upload, update and remove images representing EGI AppDB appliances. Cloudkeeper-AWS runs as a server listening for [gRPC](http://www.grpc.io/) communication usually from core [cloudkeeper](https://github.com/the-cloudkeeper-project/cloudkeeper) component.
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
            ## Requirements
         
     | 
| 
      
 12 
     | 
    
         
            +
            * Ruby >= 2.2.0
         
     | 
| 
      
 13 
     | 
    
         
            +
            * Rubygems
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
            ## Installation
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
            ### From RubyGems.org
         
     | 
| 
      
 18 
     | 
    
         
            +
            To install the most recent stable version
         
     | 
| 
      
 19 
     | 
    
         
            +
            ```bash
         
     | 
| 
      
 20 
     | 
    
         
            +
            gem install cloudkeeper-aws
         
     | 
| 
      
 21 
     | 
    
         
            +
            ```
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
            ### From source
         
     | 
| 
      
 24 
     | 
    
         
            +
            **Installation from source should never be your first choice! Especially, if you are not
         
     | 
| 
      
 25 
     | 
    
         
            +
            familiar with RVM, Bundler, Rake and other dev tools for Ruby!**
         
     | 
| 
      
 26 
     | 
    
         
            +
             
     | 
| 
      
 27 
     | 
    
         
            +
            **However, if you wish to contribute to our project, this is the right way to start.**
         
     | 
| 
      
 28 
     | 
    
         
            +
             
     | 
| 
      
 29 
     | 
    
         
            +
            To build and install the bleeding edge version from master
         
     | 
| 
      
 30 
     | 
    
         
            +
             
     | 
| 
      
 31 
     | 
    
         
            +
            ```bash
         
     | 
| 
      
 32 
     | 
    
         
            +
            git clone git://github.com/the-cloudkeeper-project/cloudkeeper-aws.git
         
     | 
| 
      
 33 
     | 
    
         
            +
            cd cloudkeeper-aws
         
     | 
| 
      
 34 
     | 
    
         
            +
            gem install bundler
         
     | 
| 
      
 35 
     | 
    
         
            +
            bundle install
         
     | 
| 
      
 36 
     | 
    
         
            +
            ```
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
            ## Configuration
         
     | 
| 
      
 39 
     | 
    
         
            +
             
     | 
| 
      
 40 
     | 
    
         
            +
            ### Create a configuration file for Cloudkeeper-AWS
         
     | 
| 
      
 41 
     | 
    
         
            +
            Configuration file can be read by Cloudkeeper-AWS from these
         
     | 
| 
      
 42 
     | 
    
         
            +
            three locations:
         
     | 
| 
      
 43 
     | 
    
         
            +
             
     | 
| 
      
 44 
     | 
    
         
            +
            * `~/.cloudkeeper-aws/cloudkeeper-aws.yml`
         
     | 
| 
      
 45 
     | 
    
         
            +
            * `/etc/cloudkeeper-aws/cloudkeeper-aws.yml`
         
     | 
| 
      
 46 
     | 
    
         
            +
            * `PATH_TO_GEM_DIR/config/cloudkeeper-aws.yml`
         
     | 
| 
      
 47 
     | 
    
         
            +
             
     | 
| 
      
 48 
     | 
    
         
            +
            The default configuration file can be found at the last location
         
     | 
| 
      
 49 
     | 
    
         
            +
            `PATH_TO_GEM_DIR/config/cloudkeeper-aws.yml`.
         
     | 
| 
      
 50 
     | 
    
         
            +
             
     | 
| 
      
 51 
     | 
    
         
            +
            ## Usage
         
     | 
| 
      
 52 
     | 
    
         
            +
            Cloudkeeper-AWS is run with executable `cloudkeeper-aws`. For further assistance run `cloudkeeper-aws help sync`:
         
     | 
| 
      
 53 
     | 
    
         
            +
            ```bash
         
     | 
| 
      
 54 
     | 
    
         
            +
            Usage:
         
     | 
| 
      
 55 
     | 
    
         
            +
              cloudkeeper-aws sync
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
            Options:
         
     | 
| 
      
 58 
     | 
    
         
            +
              [--polling-timeout=N]                      # Polling timeout value in seconds
         
     | 
| 
      
 59 
     | 
    
         
            +
                                                         # Default: 3600
         
     | 
| 
      
 60 
     | 
    
         
            +
              [--polling-interval=N]                     # Polling interval value in seconds
         
     | 
| 
      
 61 
     | 
    
         
            +
                                                         # Default: 2
         
     | 
| 
      
 62 
     | 
    
         
            +
              [--bucket-name=BUCKET-NAME]                # Name of AWS bucket for storing temp image files
         
     | 
| 
      
 63 
     | 
    
         
            +
                                                         # Default: cloudkeeper-aws
         
     | 
| 
      
 64 
     | 
    
         
            +
              [--listen-address=LISTEN-ADDRESS]          # IP address gRPC server will listen on
         
     | 
| 
      
 65 
     | 
    
         
            +
                                                         # Default: 127.0.0.1:50051
         
     | 
| 
      
 66 
     | 
    
         
            +
              [--authentication], [--no-authentication]  # Client <-> server authentication
         
     | 
| 
      
 67 
     | 
    
         
            +
              [--certificate=CERTIFICATE]                # Backend's host certificate
         
     | 
| 
      
 68 
     | 
    
         
            +
                                                         # Default: /etc/grid-security/hostcert.pem
         
     | 
| 
      
 69 
     | 
    
         
            +
              [--key=KEY]                                # Backend's host key
         
     | 
| 
      
 70 
     | 
    
         
            +
                                                         # Default: /etc/grid-security/hostkey.pem
         
     | 
| 
      
 71 
     | 
    
         
            +
              [--identifier=IDENTIFIER]                  # Instance identifier
         
     | 
| 
      
 72 
     | 
    
         
            +
                                                         # Default: cloudkeeper-aws
         
     | 
| 
      
 73 
     | 
    
         
            +
              [--core-certificate=CORE-CERTIFICATE]      # Core's certificate
         
     | 
| 
      
 74 
     | 
    
         
            +
                                                         # Default: /etc/grid-security/corecert.pem
         
     | 
| 
      
 75 
     | 
    
         
            +
              [--progress], [--no-progress]              # Print progress for each import image task
         
     | 
| 
      
 76 
     | 
    
         
            +
              --logging-level=LOGGING-LEVEL              
         
     | 
| 
      
 77 
     | 
    
         
            +
                                                         # Default: ERROR
         
     | 
| 
      
 78 
     | 
    
         
            +
                                                         # Possible values: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
         
     | 
| 
      
 79 
     | 
    
         
            +
              [--logging-file=LOGGING-FILE]              # File to write logs to
         
     | 
| 
      
 80 
     | 
    
         
            +
                                                         # Default: /var/log/cloudkeeper/cloudkeeper-aws.log
         
     | 
| 
      
 81 
     | 
    
         
            +
              [--debug], [--no-debug]                    # Runs cloudkeeper in debug mode
         
     | 
| 
      
 82 
     | 
    
         
            +
             
     | 
| 
      
 83 
     | 
    
         
            +
            Runs synchronization process
         
     | 
| 
      
 84 
     | 
    
         
            +
            ```
         
     | 
| 
      
 85 
     | 
    
         
            +
             
     | 
| 
      
 86 
     | 
    
         
            +
            ## Contributing
         
     | 
| 
      
 87 
     | 
    
         
            +
            1. Fork it ( https://github.com/the-cloudkeeper-project/cloudkeeper-aws/fork )
         
     | 
| 
      
 88 
     | 
    
         
            +
            2. Create your feature branch (`git checkout -b my-new-feature`)
         
     | 
| 
      
 89 
     | 
    
         
            +
            3. Commit your changes (`git commit -am 'Add some feature'`)
         
     | 
| 
      
 90 
     | 
    
         
            +
            4. Push to the branch (`git push origin my-new-feature`)
         
     | 
| 
      
 91 
     | 
    
         
            +
            5. Create a new Pull Request
         
     | 
    
        data/Rakefile
    ADDED
    
    | 
         @@ -0,0 +1,17 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require 'rspec/core/rake_task'
         
     | 
| 
      
 2 
     | 
    
         
            +
            require 'rubocop/rake_task'
         
     | 
| 
      
 3 
     | 
    
         
            +
            require 'bundler/gem_tasks'
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            RSpec::Core::RakeTask.new(:spec)
         
     | 
| 
      
 6 
     | 
    
         
            +
            RuboCop::RakeTask.new
         
     | 
| 
      
 7 
     | 
    
         
            +
             
     | 
| 
      
 8 
     | 
    
         
            +
            task default: :spec
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
      
 10 
     | 
    
         
            +
            desc 'Run acceptance tests (RSpec + Rubocop)'
         
     | 
| 
      
 11 
     | 
    
         
            +
            task test: 'acceptance'
         
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
            desc 'Run acceptance tests (RSpec + Rubocop)'
         
     | 
| 
      
 14 
     | 
    
         
            +
            task :acceptance do |_t|
         
     | 
| 
      
 15 
     | 
    
         
            +
              Rake::Task['spec'].invoke
         
     | 
| 
      
 16 
     | 
    
         
            +
              Rake::Task['rubocop'].invoke
         
     | 
| 
      
 17 
     | 
    
         
            +
            end
         
     | 
    
        data/bin/cloudkeeper-aws
    ADDED
    
    
| 
         @@ -0,0 +1,46 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            lib = File.expand_path('lib', __dir__)
         
     | 
| 
      
 2 
     | 
    
         
            +
            $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
         
     | 
| 
      
 3 
     | 
    
         
            +
            require 'cloudkeeper/aws/version'
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            Gem::Specification.new do |spec|
         
     | 
| 
      
 6 
     | 
    
         
            +
              spec.name = 'cloudkeeper-aws'
         
     | 
| 
      
 7 
     | 
    
         
            +
              spec.version = Cloudkeeper::Aws::VERSION
         
     | 
| 
      
 8 
     | 
    
         
            +
              spec.authors = ['Dušan Baran']
         
     | 
| 
      
 9 
     | 
    
         
            +
              spec.email = ['work.dusanbaran@gmail.com']
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
              spec.summary = 'AWS backend for cloudkeeper'
         
     | 
| 
      
 12 
     | 
    
         
            +
              spec.description = 'AWS backend for cloudkeeper'
         
     | 
| 
      
 13 
     | 
    
         
            +
              spec.homepage = 'https://github.com/the-cloudkeeper-project/cloudkeeper-aws'
         
     | 
| 
      
 14 
     | 
    
         
            +
              spec.license = 'GNU General Public License v3.0'
         
     | 
| 
      
 15 
     | 
    
         
            +
             
     | 
| 
      
 16 
     | 
    
         
            +
              spec.files = `git ls-files -z`.split("\x0").reject \
         
     | 
| 
      
 17 
     | 
    
         
            +
                { |f| f.match(%r{^(test|spec|features)/}) }
         
     | 
| 
      
 18 
     | 
    
         
            +
              gem_dir = __dir__ + '/'
         
     | 
| 
      
 19 
     | 
    
         
            +
              `git submodule --quiet foreach --recursive pwd`.split($OUTPUT_RECORD_SEPARATOR).each do |submodule_path|
         
     | 
| 
      
 20 
     | 
    
         
            +
                Dir.chdir(submodule_path) do
         
     | 
| 
      
 21 
     | 
    
         
            +
                  submodule_relative_path = submodule_path.sub gem_dir, ''
         
     | 
| 
      
 22 
     | 
    
         
            +
                  `git ls-files`.split($OUTPUT_RECORD_SEPARATOR).each do |filename|
         
     | 
| 
      
 23 
     | 
    
         
            +
                    spec.files << "#{submodule_relative_path}/#{filename}"
         
     | 
| 
      
 24 
     | 
    
         
            +
                  end
         
     | 
| 
      
 25 
     | 
    
         
            +
                end
         
     | 
| 
      
 26 
     | 
    
         
            +
              end
         
     | 
| 
      
 27 
     | 
    
         
            +
              spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
         
     | 
| 
      
 28 
     | 
    
         
            +
              spec.require_paths = ['lib']
         
     | 
| 
      
 29 
     | 
    
         
            +
             
     | 
| 
      
 30 
     | 
    
         
            +
              spec.add_development_dependency 'bundler', '~> 1.16'
         
     | 
| 
      
 31 
     | 
    
         
            +
              spec.add_development_dependency 'codecov', '~> 0.1'
         
     | 
| 
      
 32 
     | 
    
         
            +
              spec.add_development_dependency 'rake', '~> 10.0'
         
     | 
| 
      
 33 
     | 
    
         
            +
              spec.add_development_dependency 'rspec', '~> 3.0'
         
     | 
| 
      
 34 
     | 
    
         
            +
              spec.add_development_dependency 'rubocop', '~> 0.54'
         
     | 
| 
      
 35 
     | 
    
         
            +
              spec.add_development_dependency 'rubocop-rspec', '~> 1.27'
         
     | 
| 
      
 36 
     | 
    
         
            +
              spec.add_development_dependency 'vcr', '~> 4.0'
         
     | 
| 
      
 37 
     | 
    
         
            +
              spec.add_development_dependency 'webmock', '~> 3.4'
         
     | 
| 
      
 38 
     | 
    
         
            +
             
     | 
| 
      
 39 
     | 
    
         
            +
              spec.add_runtime_dependency 'activesupport', '~> 5.2'
         
     | 
| 
      
 40 
     | 
    
         
            +
              spec.add_runtime_dependency 'aws-sdk-ec2', '~> 1.43'
         
     | 
| 
      
 41 
     | 
    
         
            +
              spec.add_runtime_dependency 'aws-sdk-s3', '~> 1.17'
         
     | 
| 
      
 42 
     | 
    
         
            +
              spec.add_runtime_dependency 'grpc', '~> 1.10'
         
     | 
| 
      
 43 
     | 
    
         
            +
              spec.add_runtime_dependency 'settingslogic', '~> 2.0'
         
     | 
| 
      
 44 
     | 
    
         
            +
              spec.add_runtime_dependency 'thor', '~> 0.20'
         
     | 
| 
      
 45 
     | 
    
         
            +
              spec.add_runtime_dependency 'yell', '~> 2.0'
         
     | 
| 
      
 46 
     | 
    
         
            +
            end
         
     | 
    
        data/codecov.yml
    ADDED
    
    
| 
         @@ -0,0 +1,17 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            polling-timeout: 3600 # Timeout in seconds for AWS image import task polling
         
     | 
| 
      
 2 
     | 
    
         
            +
            polling-interval: 2 # Interval in seconds for AWS image import task polling
         
     | 
| 
      
 3 
     | 
    
         
            +
            bucket-name: cloudkeeper-aws # Bucket that is used for storing images during image import task
         
     | 
| 
      
 4 
     | 
    
         
            +
            listen-address: 127.0.0.1:50051 # IP address gRPC server will listen on
         
     | 
| 
      
 5 
     | 
    
         
            +
            authentication: false # core (client) <-> backend (server) authentication (certificate, key and core-certificate options)
         
     | 
| 
      
 6 
     | 
    
         
            +
            certificate: /etc/grid-security/hostcert.pem # Backend's host certificate
         
     | 
| 
      
 7 
     | 
    
         
            +
            key: /etc/grid-security/hostkey.pem # Backend's host key
         
     | 
| 
      
 8 
     | 
    
         
            +
            identifier: cloudkeeper-aws # Instance identifier
         
     | 
| 
      
 9 
     | 
    
         
            +
            core:
         
     | 
| 
      
 10 
     | 
    
         
            +
              certificate: /etc/grid-security/corecert.pem # Core's certificate
         
     | 
| 
      
 11 
     | 
    
         
            +
            logging:
         
     | 
| 
      
 12 
     | 
    
         
            +
              level: ERROR # Logging level
         
     | 
| 
      
 13 
     | 
    
         
            +
              file: /var/log/cloudkeeper/cloudkeeper-aws.log # File to write log to. To turn off file logging leave this field empty.
         
     | 
| 
      
 14 
     | 
    
         
            +
            progress: false
         
     | 
| 
      
 15 
     | 
    
         
            +
            debug: false # Debug mode
         
     | 
| 
      
 16 
     | 
    
         
            +
            aws: # Any AWS config data can be set here
         
     | 
| 
      
 17 
     | 
    
         
            +
                region: eu-central-1
         
     | 
| 
         @@ -0,0 +1,16 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            module Cloudkeeper
         
     | 
| 
      
 2 
     | 
    
         
            +
              # Module for aws related cloudkeeper functionality
         
     | 
| 
      
 3 
     | 
    
         
            +
              module Aws
         
     | 
| 
      
 4 
     | 
    
         
            +
                autoload :BackendExecutor, 'cloudkeeper/aws/backend_executor'
         
     | 
| 
      
 5 
     | 
    
         
            +
                autoload :CLI, 'cloudkeeper/aws/cli'
         
     | 
| 
      
 6 
     | 
    
         
            +
                autoload :Cloud, 'cloudkeeper/aws/cloud'
         
     | 
| 
      
 7 
     | 
    
         
            +
                autoload :Settings, 'cloudkeeper/aws/settings'
         
     | 
| 
      
 8 
     | 
    
         
            +
                autoload :Errors, 'cloudkeeper/aws/errors'
         
     | 
| 
      
 9 
     | 
    
         
            +
                autoload :ImageDownloader, 'cloudkeeper/aws/image_downloader'
         
     | 
| 
      
 10 
     | 
    
         
            +
                autoload :CoreConnector, 'cloudkeeper/aws/core_connector'
         
     | 
| 
      
 11 
     | 
    
         
            +
                autoload :FilterHelper, 'cloudkeeper/aws/filter_helper'
         
     | 
| 
      
 12 
     | 
    
         
            +
                autoload :ProtoHelper, 'cloudkeeper/aws/proto_helper'
         
     | 
| 
      
 13 
     | 
    
         
            +
              end
         
     | 
| 
      
 14 
     | 
    
         
            +
            end
         
     | 
| 
      
 15 
     | 
    
         
            +
             
     | 
| 
      
 16 
     | 
    
         
            +
            require 'cloudkeeper/aws/version'
         
     | 
| 
         @@ -0,0 +1,76 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            module Cloudkeeper
         
     | 
| 
      
 2 
     | 
    
         
            +
              module Aws
         
     | 
| 
      
 3 
     | 
    
         
            +
                # Module handling complex operations on cloud backend
         
     | 
| 
      
 4 
     | 
    
         
            +
                module BackendExecutor
         
     | 
| 
      
 5 
     | 
    
         
            +
                  def upload_local_appliance(appliance)
         
     | 
| 
      
 6 
     | 
    
         
            +
                    cloud.upload_file(appliance.identifier, appliance.image.location)
         
     | 
| 
      
 7 
     | 
    
         
            +
                  end
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
                  def upload_remote_appliance(appliance)
         
     | 
| 
      
 10 
     | 
    
         
            +
                    cloud.upload_data(appliance.identifier) do |write_stream|
         
     | 
| 
      
 11 
     | 
    
         
            +
                      ImageDownloader.download(appliance.image.location,
         
     | 
| 
      
 12 
     | 
    
         
            +
                                               appliance.image.username,
         
     | 
| 
      
 13 
     | 
    
         
            +
                                               appliance.image.password) do |image_segment|
         
     | 
| 
      
 14 
     | 
    
         
            +
                                                 write_stream << image_segment
         
     | 
| 
      
 15 
     | 
    
         
            +
                                               end
         
     | 
| 
      
 16 
     | 
    
         
            +
                    end
         
     | 
| 
      
 17 
     | 
    
         
            +
                  end
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
                  def upload_appliance(appliance)
         
     | 
| 
      
 20 
     | 
    
         
            +
                    upload_remote_appliance(appliance) if appliance.image.mode == :REMOTE
         
     | 
| 
      
 21 
     | 
    
         
            +
                    upload_local_appliance(appliance) if appliance.image.mode == :LOCAL
         
     | 
| 
      
 22 
     | 
    
         
            +
                  end
         
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
      
 24 
     | 
    
         
            +
                  def register_appliance(appliance)
         
     | 
| 
      
 25 
     | 
    
         
            +
                    upload_appliance(appliance)
         
     | 
| 
      
 26 
     | 
    
         
            +
                    image_id = cloud.poll_import_task(cloud.start_import_image(appliance))
         
     | 
| 
      
 27 
     | 
    
         
            +
                    cloud.set_tags(ProtoHelper.appliance_to_tags(appliance), image_id)
         
     | 
| 
      
 28 
     | 
    
         
            +
                  ensure
         
     | 
| 
      
 29 
     | 
    
         
            +
                    cloud.delete_data(appliance.identifier)
         
     | 
| 
      
 30 
     | 
    
         
            +
                  end
         
     | 
| 
      
 31 
     | 
    
         
            +
             
     | 
| 
      
 32 
     | 
    
         
            +
                  def deregister_image(appliance)
         
     | 
| 
      
 33 
     | 
    
         
            +
                    logger.debug { "Deregistering appliance #{appliance.identifier}" }
         
     | 
| 
      
 34 
     | 
    
         
            +
                    image = cloud.find_appliance(appliance.identifier)
         
     | 
| 
      
 35 
     | 
    
         
            +
                    cloud.deregister_image(image.image_id)
         
     | 
| 
      
 36 
     | 
    
         
            +
                  end
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
                  def modify_appliance(appliance)
         
     | 
| 
      
 39 
     | 
    
         
            +
                    deregister_image(appliance)
         
     | 
| 
      
 40 
     | 
    
         
            +
                    register_appliance(appliance)
         
     | 
| 
      
 41 
     | 
    
         
            +
                  end
         
     | 
| 
      
 42 
     | 
    
         
            +
             
     | 
| 
      
 43 
     | 
    
         
            +
                  def change_tags(appliance)
         
     | 
| 
      
 44 
     | 
    
         
            +
                    image = cloud.find_appliance(appliance.identifier)
         
     | 
| 
      
 45 
     | 
    
         
            +
                    cloud.set_tags(ProtoHelper.appliance_to_tags(appliance), image.image_id)
         
     | 
| 
      
 46 
     | 
    
         
            +
                  end
         
     | 
| 
      
 47 
     | 
    
         
            +
             
     | 
| 
      
 48 
     | 
    
         
            +
                  def deregister_image_list(image_list_identifier)
         
     | 
| 
      
 49 
     | 
    
         
            +
                    images = cloud.search_images(FilterHelper.image_list(image_list_identifier.image_list_identifier))
         
     | 
| 
      
 50 
     | 
    
         
            +
                    images.each { |image| cloud.deregister_image(image.image_id) }
         
     | 
| 
      
 51 
     | 
    
         
            +
                  end
         
     | 
| 
      
 52 
     | 
    
         
            +
             
     | 
| 
      
 53 
     | 
    
         
            +
                  def list_image_lists
         
     | 
| 
      
 54 
     | 
    
         
            +
                    images = cloud.search_images(FilterHelper.all_image_lists)
         
     | 
| 
      
 55 
     | 
    
         
            +
                    image_list_identifiers = images.map do |image|
         
     | 
| 
      
 56 
     | 
    
         
            +
                      image.tags.find { |tag| tag['key'] == FilterHelper::TAG_APPLIANCE_IMAGE_LIST_IDENTIFIER }['value']
         
     | 
| 
      
 57 
     | 
    
         
            +
                    end
         
     | 
| 
      
 58 
     | 
    
         
            +
                    image_list_identifiers.uniq.map { |ili| CloudkeeperGrpc::ImageListIdentifier.new(image_list_identifier: ili) }
         
     | 
| 
      
 59 
     | 
    
         
            +
                  end
         
     | 
| 
      
 60 
     | 
    
         
            +
             
     | 
| 
      
 61 
     | 
    
         
            +
                  def fetch_appliances(image_list_identifier)
         
     | 
| 
      
 62 
     | 
    
         
            +
                    images = cloud.search_images(FilterHelper.image_list(image_list_identifier.image_list_identifier))
         
     | 
| 
      
 63 
     | 
    
         
            +
                    images.map { |image| ProtoHelper.appliance_from_tags(image.tags) }
         
     | 
| 
      
 64 
     | 
    
         
            +
                  end
         
     | 
| 
      
 65 
     | 
    
         
            +
             
     | 
| 
      
 66 
     | 
    
         
            +
                  def deregister_expired_appliances
         
     | 
| 
      
 67 
     | 
    
         
            +
                    images = cloud.search_images(FilterHelper.cloudkeeper_instance)
         
     | 
| 
      
 68 
     | 
    
         
            +
                    appliances = images.map { |image| ProtoHelper.appliance_from_tags(image.tags) }
         
     | 
| 
      
 69 
     | 
    
         
            +
                    appliances.keep_if { |appliance| appliance.expiration_date <= Time.now.to_i }
         
     | 
| 
      
 70 
     | 
    
         
            +
             
     | 
| 
      
 71 
     | 
    
         
            +
                    logger.debug { "Expired appliances #{appliances.map(&:identifier).inspect}" }
         
     | 
| 
      
 72 
     | 
    
         
            +
                    appliances.each { |expired| deregister_image(expired) }
         
     | 
| 
      
 73 
     | 
    
         
            +
                  end
         
     | 
| 
      
 74 
     | 
    
         
            +
                end
         
     | 
| 
      
 75 
     | 
    
         
            +
              end
         
     | 
| 
      
 76 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,159 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require 'thor'
         
     | 
| 
      
 2 
     | 
    
         
            +
            require 'yell'
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            module Cloudkeeper
         
     | 
| 
      
 5 
     | 
    
         
            +
              module Aws
         
     | 
| 
      
 6 
     | 
    
         
            +
                # Class defining CLI of cloudkeeper-aws
         
     | 
| 
      
 7 
     | 
    
         
            +
                class CLI < Thor
         
     | 
| 
      
 8 
     | 
    
         
            +
                  SIGINT = 2
         
     | 
| 
      
 9 
     | 
    
         
            +
                  SIGTERM = 15
         
     | 
| 
      
 10 
     | 
    
         
            +
                  SIGNALS = [SIGTERM, SIGINT].freeze
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
                  class_option :'logging-level',
         
     | 
| 
      
 13 
     | 
    
         
            +
                               required: true,
         
     | 
| 
      
 14 
     | 
    
         
            +
                               default: Cloudkeeper::Aws::Settings['logging']['level'],
         
     | 
| 
      
 15 
     | 
    
         
            +
                               type: :string,
         
     | 
| 
      
 16 
     | 
    
         
            +
                               enum: Yell::Severities
         
     | 
| 
      
 17 
     | 
    
         
            +
                  class_option :'logging-file',
         
     | 
| 
      
 18 
     | 
    
         
            +
                               default: Cloudkeeper::Aws::Settings['logging']['file'],
         
     | 
| 
      
 19 
     | 
    
         
            +
                               type: :string,
         
     | 
| 
      
 20 
     | 
    
         
            +
                               desc: 'File to write logs to'
         
     | 
| 
      
 21 
     | 
    
         
            +
                  class_option :debug,
         
     | 
| 
      
 22 
     | 
    
         
            +
                               default: Cloudkeeper::Aws::Settings['debug'],
         
     | 
| 
      
 23 
     | 
    
         
            +
                               type: :boolean,
         
     | 
| 
      
 24 
     | 
    
         
            +
                               desc: 'Runs cloudkeeper in debug mode'
         
     | 
| 
      
 25 
     | 
    
         
            +
             
     | 
| 
      
 26 
     | 
    
         
            +
                  method_option :'polling-timeout',
         
     | 
| 
      
 27 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['polling-timeout'],
         
     | 
| 
      
 28 
     | 
    
         
            +
                                type: :numeric,
         
     | 
| 
      
 29 
     | 
    
         
            +
                                desc: 'Polling timeout value in seconds'
         
     | 
| 
      
 30 
     | 
    
         
            +
                  method_option :'polling-interval',
         
     | 
| 
      
 31 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['polling-interval'],
         
     | 
| 
      
 32 
     | 
    
         
            +
                                type: :numeric,
         
     | 
| 
      
 33 
     | 
    
         
            +
                                desc: 'Polling interval value in seconds'
         
     | 
| 
      
 34 
     | 
    
         
            +
                  method_option :'bucket-name',
         
     | 
| 
      
 35 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['bucket-name'],
         
     | 
| 
      
 36 
     | 
    
         
            +
                                type: :string,
         
     | 
| 
      
 37 
     | 
    
         
            +
                                desc: 'Name of AWS bucket for storing temp image files'
         
     | 
| 
      
 38 
     | 
    
         
            +
                  method_option :'listen-address',
         
     | 
| 
      
 39 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['listen-address'],
         
     | 
| 
      
 40 
     | 
    
         
            +
                                type: :string,
         
     | 
| 
      
 41 
     | 
    
         
            +
                                desc: 'IP address gRPC server will listen on'
         
     | 
| 
      
 42 
     | 
    
         
            +
                  method_option :authentication,
         
     | 
| 
      
 43 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['authentication'],
         
     | 
| 
      
 44 
     | 
    
         
            +
                                type: :boolean,
         
     | 
| 
      
 45 
     | 
    
         
            +
                                desc: 'Client <-> server authentication'
         
     | 
| 
      
 46 
     | 
    
         
            +
                  method_option :certificate,
         
     | 
| 
      
 47 
     | 
    
         
            +
                                required: false,
         
     | 
| 
      
 48 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['certificate'],
         
     | 
| 
      
 49 
     | 
    
         
            +
                                type: :string,
         
     | 
| 
      
 50 
     | 
    
         
            +
                                desc: "Backend's host certificate"
         
     | 
| 
      
 51 
     | 
    
         
            +
                  method_option :key,
         
     | 
| 
      
 52 
     | 
    
         
            +
                                required: false,
         
     | 
| 
      
 53 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['key'],
         
     | 
| 
      
 54 
     | 
    
         
            +
                                type: :string,
         
     | 
| 
      
 55 
     | 
    
         
            +
                                desc: "Backend's host key"
         
     | 
| 
      
 56 
     | 
    
         
            +
                  method_option :identifier,
         
     | 
| 
      
 57 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['identifier'],
         
     | 
| 
      
 58 
     | 
    
         
            +
                                type: :string,
         
     | 
| 
      
 59 
     | 
    
         
            +
                                desc: 'Instance identifier'
         
     | 
| 
      
 60 
     | 
    
         
            +
                  method_option :'core-certificate',
         
     | 
| 
      
 61 
     | 
    
         
            +
                                required: false,
         
     | 
| 
      
 62 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['core']['certificate'],
         
     | 
| 
      
 63 
     | 
    
         
            +
                                type: :string,
         
     | 
| 
      
 64 
     | 
    
         
            +
                                desc: "Core's certificate"
         
     | 
| 
      
 65 
     | 
    
         
            +
                  method_option :progress,
         
     | 
| 
      
 66 
     | 
    
         
            +
                                default: Cloudkeeper::Aws::Settings['progress'],
         
     | 
| 
      
 67 
     | 
    
         
            +
                                type: :boolean,
         
     | 
| 
      
 68 
     | 
    
         
            +
                                desc: 'Print progress for each import image task'
         
     | 
| 
      
 69 
     | 
    
         
            +
             
     | 
| 
      
 70 
     | 
    
         
            +
                  desc 'sync', 'Runs synchronization process'
         
     | 
| 
      
 71 
     | 
    
         
            +
                  def sync
         
     | 
| 
      
 72 
     | 
    
         
            +
                    initialize_config
         
     | 
| 
      
 73 
     | 
    
         
            +
                    initialize_logger
         
     | 
| 
      
 74 
     | 
    
         
            +
                    logger.debug { "Running with config: #{Cloudkeeper::Aws::Settings.to_hash.inspect}" }
         
     | 
| 
      
 75 
     | 
    
         
            +
                    initialize_grpc
         
     | 
| 
      
 76 
     | 
    
         
            +
                  rescue Cloudkeeper::Aws::Errors::InvalidConfigurationError => ex
         
     | 
| 
      
 77 
     | 
    
         
            +
                    abort ex.message
         
     | 
| 
      
 78 
     | 
    
         
            +
                  rescue StandardError => ex
         
     | 
| 
      
 79 
     | 
    
         
            +
                    logger.error "Unexpected error: #{ex.message}"
         
     | 
| 
      
 80 
     | 
    
         
            +
                    raise ex
         
     | 
| 
      
 81 
     | 
    
         
            +
                  end
         
     | 
| 
      
 82 
     | 
    
         
            +
             
     | 
| 
      
 83 
     | 
    
         
            +
                  desc 'version', 'Prints Cloudkeeper-AWS version'
         
     | 
| 
      
 84 
     | 
    
         
            +
                  def version
         
     | 
| 
      
 85 
     | 
    
         
            +
                    $stdout.puts Cloudkeeper::Aws::VERSION
         
     | 
| 
      
 86 
     | 
    
         
            +
                  end
         
     | 
| 
      
 87 
     | 
    
         
            +
             
     | 
| 
      
 88 
     | 
    
         
            +
                  default_task :sync
         
     | 
| 
      
 89 
     | 
    
         
            +
             
     | 
| 
      
 90 
     | 
    
         
            +
                  private
         
     | 
| 
      
 91 
     | 
    
         
            +
             
     | 
| 
      
 92 
     | 
    
         
            +
                  def initialize_grpc
         
     | 
| 
      
 93 
     | 
    
         
            +
                    grpc_server = GRPC::RpcServer.new
         
     | 
| 
      
 94 
     | 
    
         
            +
                    grpc_server.add_http2_port Cloudkeeper::Aws::Settings[:'listen-address'], credentials
         
     | 
| 
      
 95 
     | 
    
         
            +
                    grpc_server.handle Cloudkeeper::Aws::CoreConnector.new(Cloudkeeper::Aws::Cloud.new)
         
     | 
| 
      
 96 
     | 
    
         
            +
                    grpc_server.run_till_terminated
         
     | 
| 
      
 97 
     | 
    
         
            +
                  rescue SignalException => ex
         
     | 
| 
      
 98 
     | 
    
         
            +
                    raise ex unless SIGNALS.include? ex.signo
         
     | 
| 
      
 99 
     | 
    
         
            +
             
     | 
| 
      
 100 
     | 
    
         
            +
                    grpc_server.stop
         
     | 
| 
      
 101 
     | 
    
         
            +
                  end
         
     | 
| 
      
 102 
     | 
    
         
            +
             
     | 
| 
      
 103 
     | 
    
         
            +
                  def initialize_config
         
     | 
| 
      
 104 
     | 
    
         
            +
                    aws_config = Cloudkeeper::Aws::Settings[:aws]
         
     | 
| 
      
 105 
     | 
    
         
            +
                    Cloudkeeper::Aws::Settings.clear
         
     | 
| 
      
 106 
     | 
    
         
            +
                    Cloudkeeper::Aws::Settings.merge! options.to_hash
         
     | 
| 
      
 107 
     | 
    
         
            +
                    Cloudkeeper::Aws::Settings[:aws] = aws_config
         
     | 
| 
      
 108 
     | 
    
         
            +
                  end
         
     | 
| 
      
 109 
     | 
    
         
            +
             
     | 
| 
      
 110 
     | 
    
         
            +
                  def credentials
         
     | 
| 
      
 111 
     | 
    
         
            +
                    return :this_port_is_insecure unless Cloudkeeper::Aws::Settings[:authentication]
         
     | 
| 
      
 112 
     | 
    
         
            +
             
     | 
| 
      
 113 
     | 
    
         
            +
                    GRPC::Core::ServerCredentials.new(
         
     | 
| 
      
 114 
     | 
    
         
            +
                      File.read(Cloudkeeper::Aws::Settings[:'core-certificate']),
         
     | 
| 
      
 115 
     | 
    
         
            +
                      [private_key: File.read(Cloudkeeper::Aws::Settings[:key]),
         
     | 
| 
      
 116 
     | 
    
         
            +
                       cert_chain: File.read(Cloudkeeper::Aws::Settings[:certificate])],
         
     | 
| 
      
 117 
     | 
    
         
            +
                      true
         
     | 
| 
      
 118 
     | 
    
         
            +
                    )
         
     | 
| 
      
 119 
     | 
    
         
            +
                  end
         
     | 
| 
      
 120 
     | 
    
         
            +
             
     | 
| 
      
 121 
     | 
    
         
            +
                  def validate_configuration!
         
     | 
| 
      
 122 
     | 
    
         
            +
                    validate_configuration_group! :authentication,
         
     | 
| 
      
 123 
     | 
    
         
            +
                                                  %i[certificate key core-certificate],
         
     | 
| 
      
 124 
     | 
    
         
            +
                                                  'Authentication configuration missing'
         
     | 
| 
      
 125 
     | 
    
         
            +
                  end
         
     | 
| 
      
 126 
     | 
    
         
            +
             
     | 
| 
      
 127 
     | 
    
         
            +
                  def validate_configuration_group!(flag, required_options, error_message)
         
     | 
| 
      
 128 
     | 
    
         
            +
                    return unless Cloudkeeper::Aws::Settings[flag]
         
     | 
| 
      
 129 
     | 
    
         
            +
             
     | 
| 
      
 130 
     | 
    
         
            +
                    raise Cloudkeeper::Aws::Errors::InvalidConfigurationError, error_message unless all_options_available(required_options)
         
     | 
| 
      
 131 
     | 
    
         
            +
                  end
         
     | 
| 
      
 132 
     | 
    
         
            +
             
     | 
| 
      
 133 
     | 
    
         
            +
                  def all_options_available(required_options)
         
     | 
| 
      
 134 
     | 
    
         
            +
                    required_options.reduce(true) { |acc, elem| Cloudkeeper::Aws::Settings[elem] && acc }
         
     | 
| 
      
 135 
     | 
    
         
            +
                  end
         
     | 
| 
      
 136 
     | 
    
         
            +
             
     | 
| 
      
 137 
     | 
    
         
            +
                  def initialize_logger
         
     | 
| 
      
 138 
     | 
    
         
            +
                    logging_level = options['logging-level']
         
     | 
| 
      
 139 
     | 
    
         
            +
                    logging_level = 'debug' if options['debug']
         
     | 
| 
      
 140 
     | 
    
         
            +
             
     | 
| 
      
 141 
     | 
    
         
            +
                    Yell.new :stdout, name: Object, level: logging_level.downcase, format: Yell::DefaultFormat
         
     | 
| 
      
 142 
     | 
    
         
            +
                    Object.send :include, Yell::Loggable
         
     | 
| 
      
 143 
     | 
    
         
            +
             
     | 
| 
      
 144 
     | 
    
         
            +
                    setup_file_logger if options['logging-file']
         
     | 
| 
      
 145 
     | 
    
         
            +
             
     | 
| 
      
 146 
     | 
    
         
            +
                    logger.debug { 'Running in debug mode...' }
         
     | 
| 
      
 147 
     | 
    
         
            +
                  end
         
     | 
| 
      
 148 
     | 
    
         
            +
             
     | 
| 
      
 149 
     | 
    
         
            +
                  def setup_file_logger
         
     | 
| 
      
 150 
     | 
    
         
            +
                    logging_file = options['logging-file']
         
     | 
| 
      
 151 
     | 
    
         
            +
                    unless (File.exist?(logging_file) && File.writable?(logging_file)) || File.writable?(File.dirname(logging_file))
         
     | 
| 
      
 152 
     | 
    
         
            +
                      logger.error "File #{logging_file} isn't writable"
         
     | 
| 
      
 153 
     | 
    
         
            +
                      return
         
     | 
| 
      
 154 
     | 
    
         
            +
                    end
         
     | 
| 
      
 155 
     | 
    
         
            +
                    logger.adapter :file, logging_file
         
     | 
| 
      
 156 
     | 
    
         
            +
                  end
         
     | 
| 
      
 157 
     | 
    
         
            +
                end
         
     | 
| 
      
 158 
     | 
    
         
            +
              end
         
     | 
| 
      
 159 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,173 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require 'aws-sdk-s3'
         
     | 
| 
      
 2 
     | 
    
         
            +
            require 'aws-sdk-ec2'
         
     | 
| 
      
 3 
     | 
    
         
            +
            require 'timeout'
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            module Cloudkeeper
         
     | 
| 
      
 6 
     | 
    
         
            +
              module Aws
         
     | 
| 
      
 7 
     | 
    
         
            +
                # Class for AWS Cloud related operations
         
     | 
| 
      
 8 
     | 
    
         
            +
                class Cloud
         
     | 
| 
      
 9 
     | 
    
         
            +
                  attr_reader :s3, :bucket, :ec2
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                  SUCCESSFUL_STATUS = %w[completed].freeze
         
     | 
| 
      
 12 
     | 
    
         
            +
                  UNSUCCESSFUL_STATUS = %w[deleted].freeze
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
                  # Constructs Cloud object that can communicate with AWS cloud.
         
     | 
| 
      
 15 
     | 
    
         
            +
                  #
         
     | 
| 
      
 16 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 17 
     | 
    
         
            +
                  def initialize(s3service: nil, ec2service: nil)
         
     | 
| 
      
 18 
     | 
    
         
            +
                    ::Aws.config.update(Cloudkeeper::Aws::Settings['aws'].deep_symbolize_keys)
         
     | 
| 
      
 19 
     | 
    
         
            +
                    @s3 = s3service || ::Aws::S3::Resource.new
         
     | 
| 
      
 20 
     | 
    
         
            +
                    @ec2 = ec2service || ::Aws::EC2::Client.new
         
     | 
| 
      
 21 
     | 
    
         
            +
                    @bucket = s3.bucket(Cloudkeeper::Aws::Settings['bucket-name'])
         
     | 
| 
      
 22 
     | 
    
         
            +
                    bucket.create unless bucket.exists?
         
     | 
| 
      
 23 
     | 
    
         
            +
                  end
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
                  # Uploads data in block AWS file with given name
         
     | 
| 
      
 26 
     | 
    
         
            +
                  #
         
     | 
| 
      
 27 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 28 
     | 
    
         
            +
                  # @param file_name [String] key of object in bucket
         
     | 
| 
      
 29 
     | 
    
         
            +
                  # @yield [write_stream] output stream
         
     | 
| 
      
 30 
     | 
    
         
            +
                  # @raise [Cloudkeeper::Aws::Errors::BackendError] if file already exists
         
     | 
| 
      
 31 
     | 
    
         
            +
                  def upload_data(file_name, &block)
         
     | 
| 
      
 32 
     | 
    
         
            +
                    logger.debug { "Block uploading to entry (#{file_name}) in bucket(#{Cloudkeeper::Aws::Settings['bucket-name']})" }
         
     | 
| 
      
 33 
     | 
    
         
            +
                    obj = bucket.object(file_name)
         
     | 
| 
      
 34 
     | 
    
         
            +
                    if obj.exists?
         
     | 
| 
      
 35 
     | 
    
         
            +
                      raise Cloudkeeper::Aws::Errors::Backend::BackendError,
         
     | 
| 
      
 36 
     | 
    
         
            +
                            "File #{file_name} in AWS bucket already exists"
         
     | 
| 
      
 37 
     | 
    
         
            +
                    end
         
     | 
| 
      
 38 
     | 
    
         
            +
                    obj.upload_stream(&block)
         
     | 
| 
      
 39 
     | 
    
         
            +
                  end
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
                  # Uploads file to AWS bucket
         
     | 
| 
      
 42 
     | 
    
         
            +
                  #
         
     | 
| 
      
 43 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 44 
     | 
    
         
            +
                  # @param file_name [String] name of file in AWS bucket
         
     | 
| 
      
 45 
     | 
    
         
            +
                  # @param file_path [String] name of file on local machine
         
     | 
| 
      
 46 
     | 
    
         
            +
                  # @raise [Cloudkeeper::Aws::Errors::BackendError] if file already exists
         
     | 
| 
      
 47 
     | 
    
         
            +
                  def upload_file(file_name, file_path)
         
     | 
| 
      
 48 
     | 
    
         
            +
                    logger.debug { "Local file uploading to entry (#{file_name}) in bucket(#{Cloudkeeper::Aws::Settings['bucket-name']})" }
         
     | 
| 
      
 49 
     | 
    
         
            +
                    obj = bucket.object(file_name)
         
     | 
| 
      
 50 
     | 
    
         
            +
                    if obj.exists?
         
     | 
| 
      
 51 
     | 
    
         
            +
                      raise Cloudkeeper::Aws::Errors::Backend::BackendError,
         
     | 
| 
      
 52 
     | 
    
         
            +
                            "File #{file_name} in AWS bucket already exists"
         
     | 
| 
      
 53 
     | 
    
         
            +
                    end
         
     | 
| 
      
 54 
     | 
    
         
            +
                    obj.upload_file(file_path)
         
     | 
| 
      
 55 
     | 
    
         
            +
                  end
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
                  def delete_data(file_name)
         
     | 
| 
      
 58 
     | 
    
         
            +
                    logger.debug { "Deleting file: #{file_name} from bucket: #{Cloudkeeper::Aws::Settings['bucket-name']}" }
         
     | 
| 
      
 59 
     | 
    
         
            +
                    obj = bucket.object(file_name)
         
     | 
| 
      
 60 
     | 
    
         
            +
                    obj.exists? ? obj.delete : logger.info("File does not exist: #{file_name}")
         
     | 
| 
      
 61 
     | 
    
         
            +
                  end
         
     | 
| 
      
 62 
     | 
    
         
            +
             
     | 
| 
      
 63 
     | 
    
         
            +
                  # Creates import image task on AWS cloud. This task needs to be
         
     | 
| 
      
 64 
     | 
    
         
            +
                  # polled for. See {#poll_import_task}.
         
     | 
| 
      
 65 
     | 
    
         
            +
                  #
         
     | 
| 
      
 66 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 67 
     | 
    
         
            +
                  # @param appliance [Appliance] data about image
         
     | 
| 
      
 68 
     | 
    
         
            +
                  # @return [Number] import task id
         
     | 
| 
      
 69 
     | 
    
         
            +
                  def start_import_image(appliance)
         
     | 
| 
      
 70 
     | 
    
         
            +
                    logger.debug { "Starting import image task for #{appliance.identifier}" }
         
     | 
| 
      
 71 
     | 
    
         
            +
                    ec2.import_image(
         
     | 
| 
      
 72 
     | 
    
         
            +
                      description: appliance.description,
         
     | 
| 
      
 73 
     | 
    
         
            +
                      disk_containers: [disk_container(appliance)]
         
     | 
| 
      
 74 
     | 
    
         
            +
                    ).import_task_id
         
     | 
| 
      
 75 
     | 
    
         
            +
                  end
         
     | 
| 
      
 76 
     | 
    
         
            +
             
     | 
| 
      
 77 
     | 
    
         
            +
                  # Method used for generating disk container for import image task
         
     | 
| 
      
 78 
     | 
    
         
            +
                  #
         
     | 
| 
      
 79 
     | 
    
         
            +
                  # @param appliance [Appliance] data about image
         
     | 
| 
      
 80 
     | 
    
         
            +
                  # @return [Hash] disk container hash
         
     | 
| 
      
 81 
     | 
    
         
            +
                  def disk_container(appliance)
         
     | 
| 
      
 82 
     | 
    
         
            +
                    {
         
     | 
| 
      
 83 
     | 
    
         
            +
                      description: appliance.description,
         
     | 
| 
      
 84 
     | 
    
         
            +
                      format: appliance.image.format,
         
     | 
| 
      
 85 
     | 
    
         
            +
                      user_bucket: {
         
     | 
| 
      
 86 
     | 
    
         
            +
                        s3_bucket: @bucket.name,
         
     | 
| 
      
 87 
     | 
    
         
            +
                        s3_key: appliance.identifier
         
     | 
| 
      
 88 
     | 
    
         
            +
                      }
         
     | 
| 
      
 89 
     | 
    
         
            +
                    }
         
     | 
| 
      
 90 
     | 
    
         
            +
                  end
         
     | 
| 
      
 91 
     | 
    
         
            +
             
     | 
| 
      
 92 
     | 
    
         
            +
                  # Polls for import image task result. This method is blocking, so
         
     | 
| 
      
 93 
     | 
    
         
            +
                  # after image import task is completed, successfully or not, it will
         
     | 
| 
      
 94 
     | 
    
         
            +
                  # return true or false.
         
     | 
| 
      
 95 
     | 
    
         
            +
                  #
         
     | 
| 
      
 96 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 97 
     | 
    
         
            +
                  # @param import_id [String] id of import image task
         
     | 
| 
      
 98 
     | 
    
         
            +
                  # @raise [Cloudkeeper::Aws::Errors::BackendError] if polling timed out
         
     | 
| 
      
 99 
     | 
    
         
            +
                  def poll_import_task(import_id)
         
     | 
| 
      
 100 
     | 
    
         
            +
                    logger.debug { "Polling for import task #{import_id}" }
         
     | 
| 
      
 101 
     | 
    
         
            +
                    timeout do
         
     | 
| 
      
 102 
     | 
    
         
            +
                      sleep_loop do
         
     | 
| 
      
 103 
     | 
    
         
            +
                        import_task = ec2.describe_import_image_tasks(import_task_ids: [import_id]).import_image_tasks.first
         
     | 
| 
      
 104 
     | 
    
         
            +
                        print_progress(import_task)
         
     | 
| 
      
 105 
     | 
    
         
            +
                        if UNSUCCESSFUL_STATUS.include?(import_task.status)
         
     | 
| 
      
 106 
     | 
    
         
            +
                          raise Cloudkeeper::Aws::Errors::Backend::ImageImportError,
         
     | 
| 
      
 107 
     | 
    
         
            +
                                "Import failed with status #{import_task.status} and message: #{import_task.status_message}"
         
     | 
| 
      
 108 
     | 
    
         
            +
                        end
         
     | 
| 
      
 109 
     | 
    
         
            +
                        return import_task.image_id if SUCCESSFUL_STATUS.include?(import_task.status)
         
     | 
| 
      
 110 
     | 
    
         
            +
                      end
         
     | 
| 
      
 111 
     | 
    
         
            +
                    end
         
     | 
| 
      
 112 
     | 
    
         
            +
                  end
         
     | 
| 
      
 113 
     | 
    
         
            +
             
     | 
| 
      
 114 
     | 
    
         
            +
                  def print_progress(import_task)
         
     | 
| 
      
 115 
     | 
    
         
            +
                    logger.info "Import ##{import_task.import_task_id} [#{import_task.status}] with progress #{import_task.progress}%" \
         
     | 
| 
      
 116 
     | 
    
         
            +
                      if Cloudkeeper::Aws::Settings['progress']
         
     | 
| 
      
 117 
     | 
    
         
            +
                  end
         
     | 
| 
      
 118 
     | 
    
         
            +
             
     | 
| 
      
 119 
     | 
    
         
            +
                  # Simple method used for calling block in intervals
         
     | 
| 
      
 120 
     | 
    
         
            +
                  def sleep_loop
         
     | 
| 
      
 121 
     | 
    
         
            +
                    loop do
         
     | 
| 
      
 122 
     | 
    
         
            +
                      sleep Cloudkeeper::Aws::Settings['polling-interval']
         
     | 
| 
      
 123 
     | 
    
         
            +
                      yield
         
     | 
| 
      
 124 
     | 
    
         
            +
                    end
         
     | 
| 
      
 125 
     | 
    
         
            +
                  end
         
     | 
| 
      
 126 
     | 
    
         
            +
             
     | 
| 
      
 127 
     | 
    
         
            +
                  # Simple method used for handling timeout
         
     | 
| 
      
 128 
     | 
    
         
            +
                  def timeout
         
     | 
| 
      
 129 
     | 
    
         
            +
                    Timeout.timeout(Cloudkeeper::Aws::Settings['polling-timeout'],
         
     | 
| 
      
 130 
     | 
    
         
            +
                                    Cloudkeeper::Aws::Errors::Backend::TimeoutError) do
         
     | 
| 
      
 131 
     | 
    
         
            +
                      yield
         
     | 
| 
      
 132 
     | 
    
         
            +
                    end
         
     | 
| 
      
 133 
     | 
    
         
            +
                  end
         
     | 
| 
      
 134 
     | 
    
         
            +
             
     | 
| 
      
 135 
     | 
    
         
            +
                  # Deregisters specific image.
         
     | 
| 
      
 136 
     | 
    
         
            +
                  #
         
     | 
| 
      
 137 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 138 
     | 
    
         
            +
                  # @param image_id [String] id of specific AMI
         
     | 
| 
      
 139 
     | 
    
         
            +
                  def deregister_image(image_id)
         
     | 
| 
      
 140 
     | 
    
         
            +
                    logger.debug { "Deregistering AMI #{image_id}" }
         
     | 
| 
      
 141 
     | 
    
         
            +
                    ec2.deregister_image(image_id: image_id)
         
     | 
| 
      
 142 
     | 
    
         
            +
                  end
         
     | 
| 
      
 143 
     | 
    
         
            +
             
     | 
| 
      
 144 
     | 
    
         
            +
                  # Sets tags to specific AMI.
         
     | 
| 
      
 145 
     | 
    
         
            +
                  #
         
     | 
| 
      
 146 
     | 
    
         
            +
                  # @note This method can be billed by AWS
         
     | 
| 
      
 147 
     | 
    
         
            +
                  # @param tags [Array<Hash{Symbol => String}>] array of tags to set
         
     | 
| 
      
 148 
     | 
    
         
            +
                  #   to specific AMI. Tag consists of key and value symbols
         
     | 
| 
      
 149 
     | 
    
         
            +
                  # @param image_id [String] id of specific AMI
         
     | 
| 
      
 150 
     | 
    
         
            +
                  def set_tags(tags, image_id)
         
     | 
| 
      
 151 
     | 
    
         
            +
                    logger.debug { "Setting tags for AMI #{image_id}: #{tags}" }
         
     | 
| 
      
 152 
     | 
    
         
            +
                    ec2.create_tags(resources: [image_id], tags: tags)
         
     | 
| 
      
 153 
     | 
    
         
            +
                  end
         
     | 
| 
      
 154 
     | 
    
         
            +
             
     | 
| 
      
 155 
     | 
    
         
            +
                  def search_images(filters)
         
     | 
| 
      
 156 
     | 
    
         
            +
                    logger.debug { "Searching for AMI with filters: #{filters}" }
         
     | 
| 
      
 157 
     | 
    
         
            +
                    ec2.describe_images(filters: filters).images
         
     | 
| 
      
 158 
     | 
    
         
            +
                  end
         
     | 
| 
      
 159 
     | 
    
         
            +
             
     | 
| 
      
 160 
     | 
    
         
            +
                  def find_appliance(identifier)
         
     | 
| 
      
 161 
     | 
    
         
            +
                    logger.debug { "Fetching appliance with identifier: #{identifier}" }
         
     | 
| 
      
 162 
     | 
    
         
            +
                    images = ec2.describe_images(filters: FilterHelper.appliance(identifier)).images
         
     | 
| 
      
 163 
     | 
    
         
            +
                    raise Cloudkeeper::Aws::Errors::Backend::ApplianceNotFoundError, 'Appliance not found' if images.empty?
         
     | 
| 
      
 164 
     | 
    
         
            +
             
     | 
| 
      
 165 
     | 
    
         
            +
                    if images.size > 1
         
     | 
| 
      
 166 
     | 
    
         
            +
                      raise Cloudkeeper::Aws::Errors::Backend::MultipleAppliancesFoundError,
         
     | 
| 
      
 167 
     | 
    
         
            +
                            'Multiple appliances with same identifier exist in AWS'
         
     | 
| 
      
 168 
     | 
    
         
            +
                    end
         
     | 
| 
      
 169 
     | 
    
         
            +
                    images.first
         
     | 
| 
      
 170 
     | 
    
         
            +
                  end
         
     | 
| 
      
 171 
     | 
    
         
            +
                end
         
     | 
| 
      
 172 
     | 
    
         
            +
              end
         
     | 
| 
      
 173 
     | 
    
         
            +
            end
         
     |