cloudkeeper-aws 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.gitmodules +3 -0
- data/.rubocop.yml +48 -0
- data/.travis.yml +22 -0
- data/Gemfile +5 -0
- data/LICENSE +674 -0
- data/README.md +91 -0
- data/Rakefile +17 -0
- data/bin/cloudkeeper-aws +5 -0
- data/cloudkeeper-aws.gemspec +46 -0
- data/codecov.yml +4 -0
- data/config/cloudkeeper-aws.yml +17 -0
- data/lib/cloudkeeper/aws.rb +16 -0
- data/lib/cloudkeeper/aws/backend_executor.rb +76 -0
- data/lib/cloudkeeper/aws/cli.rb +159 -0
- data/lib/cloudkeeper/aws/cloud.rb +173 -0
- data/lib/cloudkeeper/aws/core_connector.rb +108 -0
- data/lib/cloudkeeper/aws/errors.rb +11 -0
- data/lib/cloudkeeper/aws/errors/backend.rb +14 -0
- data/lib/cloudkeeper/aws/errors/backend/appliance_not_found_error.rb +9 -0
- data/lib/cloudkeeper/aws/errors/backend/backend_error.rb +9 -0
- data/lib/cloudkeeper/aws/errors/backend/image_import_error.rb +9 -0
- data/lib/cloudkeeper/aws/errors/backend/multiple_appliances_found_error.rb +9 -0
- data/lib/cloudkeeper/aws/errors/backend/timeout_error.rb +9 -0
- data/lib/cloudkeeper/aws/errors/image_download_error.rb +7 -0
- data/lib/cloudkeeper/aws/errors/invalid_configuration_error.rb +7 -0
- data/lib/cloudkeeper/aws/errors/standard_error.rb +7 -0
- data/lib/cloudkeeper/aws/filter_helper.rb +33 -0
- data/lib/cloudkeeper/aws/image_downloader.rb +52 -0
- data/lib/cloudkeeper/aws/proto_helper.rb +70 -0
- data/lib/cloudkeeper/aws/settings.rb +18 -0
- data/lib/cloudkeeper/aws/version.rb +5 -0
- data/lib/cloudkeeper_grpc.rb +7 -0
- data/lib/cloudkeeper_grpc/.gitmodules +6 -0
- data/lib/cloudkeeper_grpc/CODE_OF_CONDUCT.md +49 -0
- data/lib/cloudkeeper_grpc/LICENSE.txt +17 -0
- data/lib/cloudkeeper_grpc/README.md +9 -0
- data/lib/cloudkeeper_grpc/cloudkeeper_pb.rb +59 -0
- data/lib/cloudkeeper_grpc/cloudkeeper_services_pb.rb +31 -0
- data/lib/cloudkeeper_grpc/constants.rb +10 -0
- data/lib/cloudkeeper_grpc/protos/CODE_OF_CONDUCT.md +49 -0
- data/lib/cloudkeeper_grpc/protos/LICENSE.txt +17 -0
- data/lib/cloudkeeper_grpc/protos/README.md +3 -0
- data/lib/cloudkeeper_grpc/protos/cloudkeeper.proto +64 -0
- data/lib/cloudkeeper_grpc/status-codes/CODE_OF_CONDUCT.md +49 -0
- data/lib/cloudkeeper_grpc/status-codes/LICENSE.txt +13 -0
- data/lib/cloudkeeper_grpc/status-codes/README.md +3 -0
- data/lib/cloudkeeper_grpc/status-codes/status-codes.yml +12 -0
- metadata +303 -0
data/README.md
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
<p align="center">
|
2
|
+
<img alt="Cloudkeeper-AWS" src="https://i.imgur.com/1L2LcZ0.png" width="400"/>
|
3
|
+
</p>
|
4
|
+
|
5
|
+
# Cloudkeeper-AWS
|
6
|
+
AWS backend for [Cloudkeeper](https://github.com/the-cloudkeeper-project/cloudkeeper)
|
7
|
+
|
8
|
+
## What does Cloudkeeper-AWS do?
|
9
|
+
Cloudkeeper-AWS is able to manage [AWS](https://aws.amazon.com/) cloud - upload, update and remove images representing EGI AppDB appliances. Cloudkeeper-AWS runs as a server listening for [gRPC](http://www.grpc.io/) communication usually from core [cloudkeeper](https://github.com/the-cloudkeeper-project/cloudkeeper) component.
|
10
|
+
|
11
|
+
## Requirements
|
12
|
+
* Ruby >= 2.2.0
|
13
|
+
* Rubygems
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
### From RubyGems.org
|
18
|
+
To install the most recent stable version
|
19
|
+
```bash
|
20
|
+
gem install cloudkeeper-aws
|
21
|
+
```
|
22
|
+
|
23
|
+
### From source
|
24
|
+
**Installation from source should never be your first choice! Especially, if you are not
|
25
|
+
familiar with RVM, Bundler, Rake and other dev tools for Ruby!**
|
26
|
+
|
27
|
+
**However, if you wish to contribute to our project, this is the right way to start.**
|
28
|
+
|
29
|
+
To build and install the bleeding edge version from master
|
30
|
+
|
31
|
+
```bash
|
32
|
+
git clone git://github.com/the-cloudkeeper-project/cloudkeeper-aws.git
|
33
|
+
cd cloudkeeper-aws
|
34
|
+
gem install bundler
|
35
|
+
bundle install
|
36
|
+
```
|
37
|
+
|
38
|
+
## Configuration
|
39
|
+
|
40
|
+
### Create a configuration file for Cloudkeeper-AWS
|
41
|
+
Configuration file can be read by Cloudkeeper-AWS from these
|
42
|
+
three locations:
|
43
|
+
|
44
|
+
* `~/.cloudkeeper-aws/cloudkeeper-aws.yml`
|
45
|
+
* `/etc/cloudkeeper-aws/cloudkeeper-aws.yml`
|
46
|
+
* `PATH_TO_GEM_DIR/config/cloudkeeper-aws.yml`
|
47
|
+
|
48
|
+
The default configuration file can be found at the last location
|
49
|
+
`PATH_TO_GEM_DIR/config/cloudkeeper-aws.yml`.
|
50
|
+
|
51
|
+
## Usage
|
52
|
+
Cloudkeeper-AWS is run with executable `cloudkeeper-aws`. For further assistance run `cloudkeeper-aws help sync`:
|
53
|
+
```bash
|
54
|
+
Usage:
|
55
|
+
cloudkeeper-aws sync
|
56
|
+
|
57
|
+
Options:
|
58
|
+
[--polling-timeout=N] # Polling timeout value in seconds
|
59
|
+
# Default: 3600
|
60
|
+
[--polling-interval=N] # Polling interval value in seconds
|
61
|
+
# Default: 2
|
62
|
+
[--bucket-name=BUCKET-NAME] # Name of AWS bucket for storing temp image files
|
63
|
+
# Default: cloudkeeper-aws
|
64
|
+
[--listen-address=LISTEN-ADDRESS] # IP address gRPC server will listen on
|
65
|
+
# Default: 127.0.0.1:50051
|
66
|
+
[--authentication], [--no-authentication] # Client <-> server authentication
|
67
|
+
[--certificate=CERTIFICATE] # Backend's host certificate
|
68
|
+
# Default: /etc/grid-security/hostcert.pem
|
69
|
+
[--key=KEY] # Backend's host key
|
70
|
+
# Default: /etc/grid-security/hostkey.pem
|
71
|
+
[--identifier=IDENTIFIER] # Instance identifier
|
72
|
+
# Default: cloudkeeper-aws
|
73
|
+
[--core-certificate=CORE-CERTIFICATE] # Core's certificate
|
74
|
+
# Default: /etc/grid-security/corecert.pem
|
75
|
+
[--progress], [--no-progress] # Print progress for each import image task
|
76
|
+
--logging-level=LOGGING-LEVEL
|
77
|
+
# Default: ERROR
|
78
|
+
# Possible values: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
|
79
|
+
[--logging-file=LOGGING-FILE] # File to write logs to
|
80
|
+
# Default: /var/log/cloudkeeper/cloudkeeper-aws.log
|
81
|
+
[--debug], [--no-debug] # Runs cloudkeeper in debug mode
|
82
|
+
|
83
|
+
Runs synchronization process
|
84
|
+
```
|
85
|
+
|
86
|
+
## Contributing
|
87
|
+
1. Fork it ( https://github.com/the-cloudkeeper-project/cloudkeeper-aws/fork )
|
88
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
89
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
90
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
91
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'rspec/core/rake_task'
|
2
|
+
require 'rubocop/rake_task'
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
|
5
|
+
RSpec::Core::RakeTask.new(:spec)
|
6
|
+
RuboCop::RakeTask.new
|
7
|
+
|
8
|
+
task default: :spec
|
9
|
+
|
10
|
+
desc 'Run acceptance tests (RSpec + Rubocop)'
|
11
|
+
task test: 'acceptance'
|
12
|
+
|
13
|
+
desc 'Run acceptance tests (RSpec + Rubocop)'
|
14
|
+
task :acceptance do |_t|
|
15
|
+
Rake::Task['spec'].invoke
|
16
|
+
Rake::Task['rubocop'].invoke
|
17
|
+
end
|
data/bin/cloudkeeper-aws
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'cloudkeeper/aws/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = 'cloudkeeper-aws'
|
7
|
+
spec.version = Cloudkeeper::Aws::VERSION
|
8
|
+
spec.authors = ['Dušan Baran']
|
9
|
+
spec.email = ['work.dusanbaran@gmail.com']
|
10
|
+
|
11
|
+
spec.summary = 'AWS backend for cloudkeeper'
|
12
|
+
spec.description = 'AWS backend for cloudkeeper'
|
13
|
+
spec.homepage = 'https://github.com/the-cloudkeeper-project/cloudkeeper-aws'
|
14
|
+
spec.license = 'GNU General Public License v3.0'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject \
|
17
|
+
{ |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
+
gem_dir = __dir__ + '/'
|
19
|
+
`git submodule --quiet foreach --recursive pwd`.split($OUTPUT_RECORD_SEPARATOR).each do |submodule_path|
|
20
|
+
Dir.chdir(submodule_path) do
|
21
|
+
submodule_relative_path = submodule_path.sub gem_dir, ''
|
22
|
+
`git ls-files`.split($OUTPUT_RECORD_SEPARATOR).each do |filename|
|
23
|
+
spec.files << "#{submodule_relative_path}/#{filename}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
28
|
+
spec.require_paths = ['lib']
|
29
|
+
|
30
|
+
spec.add_development_dependency 'bundler', '~> 1.16'
|
31
|
+
spec.add_development_dependency 'codecov', '~> 0.1'
|
32
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
33
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
34
|
+
spec.add_development_dependency 'rubocop', '~> 0.54'
|
35
|
+
spec.add_development_dependency 'rubocop-rspec', '~> 1.27'
|
36
|
+
spec.add_development_dependency 'vcr', '~> 4.0'
|
37
|
+
spec.add_development_dependency 'webmock', '~> 3.4'
|
38
|
+
|
39
|
+
spec.add_runtime_dependency 'activesupport', '~> 5.2'
|
40
|
+
spec.add_runtime_dependency 'aws-sdk-ec2', '~> 1.43'
|
41
|
+
spec.add_runtime_dependency 'aws-sdk-s3', '~> 1.17'
|
42
|
+
spec.add_runtime_dependency 'grpc', '~> 1.10'
|
43
|
+
spec.add_runtime_dependency 'settingslogic', '~> 2.0'
|
44
|
+
spec.add_runtime_dependency 'thor', '~> 0.20'
|
45
|
+
spec.add_runtime_dependency 'yell', '~> 2.0'
|
46
|
+
end
|
data/codecov.yml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
polling-timeout: 3600 # Timeout in seconds for AWS image import task polling
|
2
|
+
polling-interval: 2 # Interval in seconds for AWS image import task polling
|
3
|
+
bucket-name: cloudkeeper-aws # Bucket that is used for storing images during image import task
|
4
|
+
listen-address: 127.0.0.1:50051 # IP address gRPC server will listen on
|
5
|
+
authentication: false # core (client) <-> backend (server) authentication (certificate, key and core-certificate options)
|
6
|
+
certificate: /etc/grid-security/hostcert.pem # Backend's host certificate
|
7
|
+
key: /etc/grid-security/hostkey.pem # Backend's host key
|
8
|
+
identifier: cloudkeeper-aws # Instance identifier
|
9
|
+
core:
|
10
|
+
certificate: /etc/grid-security/corecert.pem # Core's certificate
|
11
|
+
logging:
|
12
|
+
level: ERROR # Logging level
|
13
|
+
file: /var/log/cloudkeeper/cloudkeeper-aws.log # File to write log to. To turn off file logging leave this field empty.
|
14
|
+
progress: false
|
15
|
+
debug: false # Debug mode
|
16
|
+
aws: # Any AWS config data can be set here
|
17
|
+
region: eu-central-1
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Cloudkeeper
|
2
|
+
# Module for aws related cloudkeeper functionality
|
3
|
+
module Aws
|
4
|
+
autoload :BackendExecutor, 'cloudkeeper/aws/backend_executor'
|
5
|
+
autoload :CLI, 'cloudkeeper/aws/cli'
|
6
|
+
autoload :Cloud, 'cloudkeeper/aws/cloud'
|
7
|
+
autoload :Settings, 'cloudkeeper/aws/settings'
|
8
|
+
autoload :Errors, 'cloudkeeper/aws/errors'
|
9
|
+
autoload :ImageDownloader, 'cloudkeeper/aws/image_downloader'
|
10
|
+
autoload :CoreConnector, 'cloudkeeper/aws/core_connector'
|
11
|
+
autoload :FilterHelper, 'cloudkeeper/aws/filter_helper'
|
12
|
+
autoload :ProtoHelper, 'cloudkeeper/aws/proto_helper'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
require 'cloudkeeper/aws/version'
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module Cloudkeeper
|
2
|
+
module Aws
|
3
|
+
# Module handling complex operations on cloud backend
|
4
|
+
module BackendExecutor
|
5
|
+
def upload_local_appliance(appliance)
|
6
|
+
cloud.upload_file(appliance.identifier, appliance.image.location)
|
7
|
+
end
|
8
|
+
|
9
|
+
def upload_remote_appliance(appliance)
|
10
|
+
cloud.upload_data(appliance.identifier) do |write_stream|
|
11
|
+
ImageDownloader.download(appliance.image.location,
|
12
|
+
appliance.image.username,
|
13
|
+
appliance.image.password) do |image_segment|
|
14
|
+
write_stream << image_segment
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def upload_appliance(appliance)
|
20
|
+
upload_remote_appliance(appliance) if appliance.image.mode == :REMOTE
|
21
|
+
upload_local_appliance(appliance) if appliance.image.mode == :LOCAL
|
22
|
+
end
|
23
|
+
|
24
|
+
def register_appliance(appliance)
|
25
|
+
upload_appliance(appliance)
|
26
|
+
image_id = cloud.poll_import_task(cloud.start_import_image(appliance))
|
27
|
+
cloud.set_tags(ProtoHelper.appliance_to_tags(appliance), image_id)
|
28
|
+
ensure
|
29
|
+
cloud.delete_data(appliance.identifier)
|
30
|
+
end
|
31
|
+
|
32
|
+
def deregister_image(appliance)
|
33
|
+
logger.debug { "Deregistering appliance #{appliance.identifier}" }
|
34
|
+
image = cloud.find_appliance(appliance.identifier)
|
35
|
+
cloud.deregister_image(image.image_id)
|
36
|
+
end
|
37
|
+
|
38
|
+
def modify_appliance(appliance)
|
39
|
+
deregister_image(appliance)
|
40
|
+
register_appliance(appliance)
|
41
|
+
end
|
42
|
+
|
43
|
+
def change_tags(appliance)
|
44
|
+
image = cloud.find_appliance(appliance.identifier)
|
45
|
+
cloud.set_tags(ProtoHelper.appliance_to_tags(appliance), image.image_id)
|
46
|
+
end
|
47
|
+
|
48
|
+
def deregister_image_list(image_list_identifier)
|
49
|
+
images = cloud.search_images(FilterHelper.image_list(image_list_identifier.image_list_identifier))
|
50
|
+
images.each { |image| cloud.deregister_image(image.image_id) }
|
51
|
+
end
|
52
|
+
|
53
|
+
def list_image_lists
|
54
|
+
images = cloud.search_images(FilterHelper.all_image_lists)
|
55
|
+
image_list_identifiers = images.map do |image|
|
56
|
+
image.tags.find { |tag| tag['key'] == FilterHelper::TAG_APPLIANCE_IMAGE_LIST_IDENTIFIER }['value']
|
57
|
+
end
|
58
|
+
image_list_identifiers.uniq.map { |ili| CloudkeeperGrpc::ImageListIdentifier.new(image_list_identifier: ili) }
|
59
|
+
end
|
60
|
+
|
61
|
+
def fetch_appliances(image_list_identifier)
|
62
|
+
images = cloud.search_images(FilterHelper.image_list(image_list_identifier.image_list_identifier))
|
63
|
+
images.map { |image| ProtoHelper.appliance_from_tags(image.tags) }
|
64
|
+
end
|
65
|
+
|
66
|
+
def deregister_expired_appliances
|
67
|
+
images = cloud.search_images(FilterHelper.cloudkeeper_instance)
|
68
|
+
appliances = images.map { |image| ProtoHelper.appliance_from_tags(image.tags) }
|
69
|
+
appliances.keep_if { |appliance| appliance.expiration_date <= Time.now.to_i }
|
70
|
+
|
71
|
+
logger.debug { "Expired appliances #{appliances.map(&:identifier).inspect}" }
|
72
|
+
appliances.each { |expired| deregister_image(expired) }
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'yell'
|
3
|
+
|
4
|
+
module Cloudkeeper
|
5
|
+
module Aws
|
6
|
+
# Class defining CLI of cloudkeeper-aws
|
7
|
+
class CLI < Thor
|
8
|
+
SIGINT = 2
|
9
|
+
SIGTERM = 15
|
10
|
+
SIGNALS = [SIGTERM, SIGINT].freeze
|
11
|
+
|
12
|
+
class_option :'logging-level',
|
13
|
+
required: true,
|
14
|
+
default: Cloudkeeper::Aws::Settings['logging']['level'],
|
15
|
+
type: :string,
|
16
|
+
enum: Yell::Severities
|
17
|
+
class_option :'logging-file',
|
18
|
+
default: Cloudkeeper::Aws::Settings['logging']['file'],
|
19
|
+
type: :string,
|
20
|
+
desc: 'File to write logs to'
|
21
|
+
class_option :debug,
|
22
|
+
default: Cloudkeeper::Aws::Settings['debug'],
|
23
|
+
type: :boolean,
|
24
|
+
desc: 'Runs cloudkeeper in debug mode'
|
25
|
+
|
26
|
+
method_option :'polling-timeout',
|
27
|
+
default: Cloudkeeper::Aws::Settings['polling-timeout'],
|
28
|
+
type: :numeric,
|
29
|
+
desc: 'Polling timeout value in seconds'
|
30
|
+
method_option :'polling-interval',
|
31
|
+
default: Cloudkeeper::Aws::Settings['polling-interval'],
|
32
|
+
type: :numeric,
|
33
|
+
desc: 'Polling interval value in seconds'
|
34
|
+
method_option :'bucket-name',
|
35
|
+
default: Cloudkeeper::Aws::Settings['bucket-name'],
|
36
|
+
type: :string,
|
37
|
+
desc: 'Name of AWS bucket for storing temp image files'
|
38
|
+
method_option :'listen-address',
|
39
|
+
default: Cloudkeeper::Aws::Settings['listen-address'],
|
40
|
+
type: :string,
|
41
|
+
desc: 'IP address gRPC server will listen on'
|
42
|
+
method_option :authentication,
|
43
|
+
default: Cloudkeeper::Aws::Settings['authentication'],
|
44
|
+
type: :boolean,
|
45
|
+
desc: 'Client <-> server authentication'
|
46
|
+
method_option :certificate,
|
47
|
+
required: false,
|
48
|
+
default: Cloudkeeper::Aws::Settings['certificate'],
|
49
|
+
type: :string,
|
50
|
+
desc: "Backend's host certificate"
|
51
|
+
method_option :key,
|
52
|
+
required: false,
|
53
|
+
default: Cloudkeeper::Aws::Settings['key'],
|
54
|
+
type: :string,
|
55
|
+
desc: "Backend's host key"
|
56
|
+
method_option :identifier,
|
57
|
+
default: Cloudkeeper::Aws::Settings['identifier'],
|
58
|
+
type: :string,
|
59
|
+
desc: 'Instance identifier'
|
60
|
+
method_option :'core-certificate',
|
61
|
+
required: false,
|
62
|
+
default: Cloudkeeper::Aws::Settings['core']['certificate'],
|
63
|
+
type: :string,
|
64
|
+
desc: "Core's certificate"
|
65
|
+
method_option :progress,
|
66
|
+
default: Cloudkeeper::Aws::Settings['progress'],
|
67
|
+
type: :boolean,
|
68
|
+
desc: 'Print progress for each import image task'
|
69
|
+
|
70
|
+
desc 'sync', 'Runs synchronization process'
|
71
|
+
def sync
|
72
|
+
initialize_config
|
73
|
+
initialize_logger
|
74
|
+
logger.debug { "Running with config: #{Cloudkeeper::Aws::Settings.to_hash.inspect}" }
|
75
|
+
initialize_grpc
|
76
|
+
rescue Cloudkeeper::Aws::Errors::InvalidConfigurationError => ex
|
77
|
+
abort ex.message
|
78
|
+
rescue StandardError => ex
|
79
|
+
logger.error "Unexpected error: #{ex.message}"
|
80
|
+
raise ex
|
81
|
+
end
|
82
|
+
|
83
|
+
desc 'version', 'Prints Cloudkeeper-AWS version'
|
84
|
+
def version
|
85
|
+
$stdout.puts Cloudkeeper::Aws::VERSION
|
86
|
+
end
|
87
|
+
|
88
|
+
default_task :sync
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def initialize_grpc
|
93
|
+
grpc_server = GRPC::RpcServer.new
|
94
|
+
grpc_server.add_http2_port Cloudkeeper::Aws::Settings[:'listen-address'], credentials
|
95
|
+
grpc_server.handle Cloudkeeper::Aws::CoreConnector.new(Cloudkeeper::Aws::Cloud.new)
|
96
|
+
grpc_server.run_till_terminated
|
97
|
+
rescue SignalException => ex
|
98
|
+
raise ex unless SIGNALS.include? ex.signo
|
99
|
+
|
100
|
+
grpc_server.stop
|
101
|
+
end
|
102
|
+
|
103
|
+
def initialize_config
|
104
|
+
aws_config = Cloudkeeper::Aws::Settings[:aws]
|
105
|
+
Cloudkeeper::Aws::Settings.clear
|
106
|
+
Cloudkeeper::Aws::Settings.merge! options.to_hash
|
107
|
+
Cloudkeeper::Aws::Settings[:aws] = aws_config
|
108
|
+
end
|
109
|
+
|
110
|
+
def credentials
|
111
|
+
return :this_port_is_insecure unless Cloudkeeper::Aws::Settings[:authentication]
|
112
|
+
|
113
|
+
GRPC::Core::ServerCredentials.new(
|
114
|
+
File.read(Cloudkeeper::Aws::Settings[:'core-certificate']),
|
115
|
+
[private_key: File.read(Cloudkeeper::Aws::Settings[:key]),
|
116
|
+
cert_chain: File.read(Cloudkeeper::Aws::Settings[:certificate])],
|
117
|
+
true
|
118
|
+
)
|
119
|
+
end
|
120
|
+
|
121
|
+
def validate_configuration!
|
122
|
+
validate_configuration_group! :authentication,
|
123
|
+
%i[certificate key core-certificate],
|
124
|
+
'Authentication configuration missing'
|
125
|
+
end
|
126
|
+
|
127
|
+
def validate_configuration_group!(flag, required_options, error_message)
|
128
|
+
return unless Cloudkeeper::Aws::Settings[flag]
|
129
|
+
|
130
|
+
raise Cloudkeeper::Aws::Errors::InvalidConfigurationError, error_message unless all_options_available(required_options)
|
131
|
+
end
|
132
|
+
|
133
|
+
def all_options_available(required_options)
|
134
|
+
required_options.reduce(true) { |acc, elem| Cloudkeeper::Aws::Settings[elem] && acc }
|
135
|
+
end
|
136
|
+
|
137
|
+
def initialize_logger
|
138
|
+
logging_level = options['logging-level']
|
139
|
+
logging_level = 'debug' if options['debug']
|
140
|
+
|
141
|
+
Yell.new :stdout, name: Object, level: logging_level.downcase, format: Yell::DefaultFormat
|
142
|
+
Object.send :include, Yell::Loggable
|
143
|
+
|
144
|
+
setup_file_logger if options['logging-file']
|
145
|
+
|
146
|
+
logger.debug { 'Running in debug mode...' }
|
147
|
+
end
|
148
|
+
|
149
|
+
def setup_file_logger
|
150
|
+
logging_file = options['logging-file']
|
151
|
+
unless (File.exist?(logging_file) && File.writable?(logging_file)) || File.writable?(File.dirname(logging_file))
|
152
|
+
logger.error "File #{logging_file} isn't writable"
|
153
|
+
return
|
154
|
+
end
|
155
|
+
logger.adapter :file, logging_file
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
@@ -0,0 +1,173 @@
|
|
1
|
+
require 'aws-sdk-s3'
|
2
|
+
require 'aws-sdk-ec2'
|
3
|
+
require 'timeout'
|
4
|
+
|
5
|
+
module Cloudkeeper
|
6
|
+
module Aws
|
7
|
+
# Class for AWS Cloud related operations
|
8
|
+
class Cloud
|
9
|
+
attr_reader :s3, :bucket, :ec2
|
10
|
+
|
11
|
+
SUCCESSFUL_STATUS = %w[completed].freeze
|
12
|
+
UNSUCCESSFUL_STATUS = %w[deleted].freeze
|
13
|
+
|
14
|
+
# Constructs Cloud object that can communicate with AWS cloud.
|
15
|
+
#
|
16
|
+
# @note This method can be billed by AWS
|
17
|
+
def initialize(s3service: nil, ec2service: nil)
|
18
|
+
::Aws.config.update(Cloudkeeper::Aws::Settings['aws'].deep_symbolize_keys)
|
19
|
+
@s3 = s3service || ::Aws::S3::Resource.new
|
20
|
+
@ec2 = ec2service || ::Aws::EC2::Client.new
|
21
|
+
@bucket = s3.bucket(Cloudkeeper::Aws::Settings['bucket-name'])
|
22
|
+
bucket.create unless bucket.exists?
|
23
|
+
end
|
24
|
+
|
25
|
+
# Uploads data in block AWS file with given name
|
26
|
+
#
|
27
|
+
# @note This method can be billed by AWS
|
28
|
+
# @param file_name [String] key of object in bucket
|
29
|
+
# @yield [write_stream] output stream
|
30
|
+
# @raise [Cloudkeeper::Aws::Errors::BackendError] if file already exists
|
31
|
+
def upload_data(file_name, &block)
|
32
|
+
logger.debug { "Block uploading to entry (#{file_name}) in bucket(#{Cloudkeeper::Aws::Settings['bucket-name']})" }
|
33
|
+
obj = bucket.object(file_name)
|
34
|
+
if obj.exists?
|
35
|
+
raise Cloudkeeper::Aws::Errors::Backend::BackendError,
|
36
|
+
"File #{file_name} in AWS bucket already exists"
|
37
|
+
end
|
38
|
+
obj.upload_stream(&block)
|
39
|
+
end
|
40
|
+
|
41
|
+
# Uploads file to AWS bucket
|
42
|
+
#
|
43
|
+
# @note This method can be billed by AWS
|
44
|
+
# @param file_name [String] name of file in AWS bucket
|
45
|
+
# @param file_path [String] name of file on local machine
|
46
|
+
# @raise [Cloudkeeper::Aws::Errors::BackendError] if file already exists
|
47
|
+
def upload_file(file_name, file_path)
|
48
|
+
logger.debug { "Local file uploading to entry (#{file_name}) in bucket(#{Cloudkeeper::Aws::Settings['bucket-name']})" }
|
49
|
+
obj = bucket.object(file_name)
|
50
|
+
if obj.exists?
|
51
|
+
raise Cloudkeeper::Aws::Errors::Backend::BackendError,
|
52
|
+
"File #{file_name} in AWS bucket already exists"
|
53
|
+
end
|
54
|
+
obj.upload_file(file_path)
|
55
|
+
end
|
56
|
+
|
57
|
+
def delete_data(file_name)
|
58
|
+
logger.debug { "Deleting file: #{file_name} from bucket: #{Cloudkeeper::Aws::Settings['bucket-name']}" }
|
59
|
+
obj = bucket.object(file_name)
|
60
|
+
obj.exists? ? obj.delete : logger.info("File does not exist: #{file_name}")
|
61
|
+
end
|
62
|
+
|
63
|
+
# Creates import image task on AWS cloud. This task needs to be
|
64
|
+
# polled for. See {#poll_import_task}.
|
65
|
+
#
|
66
|
+
# @note This method can be billed by AWS
|
67
|
+
# @param appliance [Appliance] data about image
|
68
|
+
# @return [Number] import task id
|
69
|
+
def start_import_image(appliance)
|
70
|
+
logger.debug { "Starting import image task for #{appliance.identifier}" }
|
71
|
+
ec2.import_image(
|
72
|
+
description: appliance.description,
|
73
|
+
disk_containers: [disk_container(appliance)]
|
74
|
+
).import_task_id
|
75
|
+
end
|
76
|
+
|
77
|
+
# Method used for generating disk container for import image task
|
78
|
+
#
|
79
|
+
# @param appliance [Appliance] data about image
|
80
|
+
# @return [Hash] disk container hash
|
81
|
+
def disk_container(appliance)
|
82
|
+
{
|
83
|
+
description: appliance.description,
|
84
|
+
format: appliance.image.format,
|
85
|
+
user_bucket: {
|
86
|
+
s3_bucket: @bucket.name,
|
87
|
+
s3_key: appliance.identifier
|
88
|
+
}
|
89
|
+
}
|
90
|
+
end
|
91
|
+
|
92
|
+
# Polls for import image task result. This method is blocking, so
|
93
|
+
# after image import task is completed, successfully or not, it will
|
94
|
+
# return true or false.
|
95
|
+
#
|
96
|
+
# @note This method can be billed by AWS
|
97
|
+
# @param import_id [String] id of import image task
|
98
|
+
# @raise [Cloudkeeper::Aws::Errors::BackendError] if polling timed out
|
99
|
+
def poll_import_task(import_id)
|
100
|
+
logger.debug { "Polling for import task #{import_id}" }
|
101
|
+
timeout do
|
102
|
+
sleep_loop do
|
103
|
+
import_task = ec2.describe_import_image_tasks(import_task_ids: [import_id]).import_image_tasks.first
|
104
|
+
print_progress(import_task)
|
105
|
+
if UNSUCCESSFUL_STATUS.include?(import_task.status)
|
106
|
+
raise Cloudkeeper::Aws::Errors::Backend::ImageImportError,
|
107
|
+
"Import failed with status #{import_task.status} and message: #{import_task.status_message}"
|
108
|
+
end
|
109
|
+
return import_task.image_id if SUCCESSFUL_STATUS.include?(import_task.status)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def print_progress(import_task)
|
115
|
+
logger.info "Import ##{import_task.import_task_id} [#{import_task.status}] with progress #{import_task.progress}%" \
|
116
|
+
if Cloudkeeper::Aws::Settings['progress']
|
117
|
+
end
|
118
|
+
|
119
|
+
# Simple method used for calling block in intervals
|
120
|
+
def sleep_loop
|
121
|
+
loop do
|
122
|
+
sleep Cloudkeeper::Aws::Settings['polling-interval']
|
123
|
+
yield
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
# Simple method used for handling timeout
|
128
|
+
def timeout
|
129
|
+
Timeout.timeout(Cloudkeeper::Aws::Settings['polling-timeout'],
|
130
|
+
Cloudkeeper::Aws::Errors::Backend::TimeoutError) do
|
131
|
+
yield
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Deregisters specific image.
|
136
|
+
#
|
137
|
+
# @note This method can be billed by AWS
|
138
|
+
# @param image_id [String] id of specific AMI
|
139
|
+
def deregister_image(image_id)
|
140
|
+
logger.debug { "Deregistering AMI #{image_id}" }
|
141
|
+
ec2.deregister_image(image_id: image_id)
|
142
|
+
end
|
143
|
+
|
144
|
+
# Sets tags to specific AMI.
|
145
|
+
#
|
146
|
+
# @note This method can be billed by AWS
|
147
|
+
# @param tags [Array<Hash{Symbol => String}>] array of tags to set
|
148
|
+
# to specific AMI. Tag consists of key and value symbols
|
149
|
+
# @param image_id [String] id of specific AMI
|
150
|
+
def set_tags(tags, image_id)
|
151
|
+
logger.debug { "Setting tags for AMI #{image_id}: #{tags}" }
|
152
|
+
ec2.create_tags(resources: [image_id], tags: tags)
|
153
|
+
end
|
154
|
+
|
155
|
+
def search_images(filters)
|
156
|
+
logger.debug { "Searching for AMI with filters: #{filters}" }
|
157
|
+
ec2.describe_images(filters: filters).images
|
158
|
+
end
|
159
|
+
|
160
|
+
def find_appliance(identifier)
|
161
|
+
logger.debug { "Fetching appliance with identifier: #{identifier}" }
|
162
|
+
images = ec2.describe_images(filters: FilterHelper.appliance(identifier)).images
|
163
|
+
raise Cloudkeeper::Aws::Errors::Backend::ApplianceNotFoundError, 'Appliance not found' if images.empty?
|
164
|
+
|
165
|
+
if images.size > 1
|
166
|
+
raise Cloudkeeper::Aws::Errors::Backend::MultipleAppliancesFoundError,
|
167
|
+
'Multiple appliances with same identifier exist in AWS'
|
168
|
+
end
|
169
|
+
images.first
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|