cloudkeeper 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e3b7552e766ed9dff94b3c02ead5b2d7457146d7
+  data.tar.gz: 9fc5d03ddd65bd5502b9bbf7cfa716c5e5f17306
+SHA512:
+  metadata.gz: 0a9273fe7503544dd2ed1749d577fbf49a8db35a7b6a9dc4f71230109000bf7e302d9b5c8050a955515b105af0dbcb48fc23a8923b06f3f4f9631cce3a020cae
+  data.tar.gz: 69ee71cc79f047eeb8efb5a80c512285feca404c2047c178d00d563fc57aab294ba7db8873ece88cf038e966167f466ce295e783d366750801bf7e719ac56eb1

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+example
+/vendor/
+/coverage/

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/cloudkeeper_grpc"]
+	path = lib/cloudkeeper_grpc
+	url = https://github.com/the-cloudkeeper-project/cloudkeeper-grpc-ruby.git

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,61 @@
+require: rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.0
+  Exclude:
+    - 'vendor/**/*'
+    - 'lib/cloudkeeper_grpc/*'
+
+Metrics/LineLength:
+  Max: 135
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 15
+  Exclude:
+    - 'lib/cloudkeeper/entities/appliance.rb'
+    - 'lib/cloudkeeper/nginx/http_server.rb'
+
+Metrics/AbcSize:
+  Max: 20
+  Exclude:
+    - 'lib/cloudkeeper/entities/conversions.rb'
+
+Metrics/ClassLength:
+  Max: 120
+  Exclude:
+    - 'lib/cloudkeeper/cli.rb'
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Max: 3
+
+RSpec/NestedGroups:
+  Max: 4
+  Include:
+    - 'spec/cloudkeeper/backend_connector_spec.rb'
+
+RSpec/SubjectStub:
+  Exclude:
+    - 'spec/cloudkeeper/backend_connector_spec.rb'
+
+Metrics/ParameterLists:
+  Exclude:
+    - 'lib/cloudkeeper/entities/*.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'Rakefile'
+    - '**/*.rake'
+    - 'spec/**/*.rb'
+    - '*.gemspec'
+    - 'lib/cloudkeeper/entities/appliance.rb'
+    - 'lib/cloudkeeper/entities/image_list.rb'
+    - 'lib/cloudkeeper/grpc/*'

data/.travis.yml

@@ -0,0 +1,21 @@
+sudo: false
+
+language: ruby
+rvm:
+  - ruby-head
+  - 2.2.6
+  - 2.3.3
+  - 2.4.0
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+fast_finish: true
+
+branches:
+  only:
+    - master
+
+before_install: 'gem install bundler -v 1.13.0'
+
+script: 'bundle exec rake acceptance'

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at kimle.michal@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cloud-keeper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,17 @@
+The work represented by this source file was partially or entirely
+funded by the EGI-Engage project co-funded by the European Union (EU)
+Horizon 2020 program under Grant number 654142.
+
+Copyright (c) 2017 CESNET
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,122 @@
+# cloudkeeper
+cloudkeeper is an AppDB <-> cloud synchronization utility
+
+[![Build Status](https://secure.travis-ci.org/the-cloudkeeper-project/cloudkeeper.png)](http://travis-ci.org/the-cloudkeeper-project/cloudkeeper)
+[![Dependency Status](https://gemnasium.com/the-cloudkeeper-project/cloudkeeper.png)](https://gemnasium.com/the-cloudkeeper-project/cloudkeeper)
+[![Gem Version](https://fury-badge.herokuapp.com/rb/cloudkeeper.png)](https://badge.fury.io/rb/cloudkeeper)
+[![Code Climate](https://codeclimate.com/github/the-cloudkeeper-project/cloudkeeper.png)](https://codeclimate.com/github/the-cloudkeeper-project/cloudkeeper)
+
+## What does cloudkeeper do?
+cloudkeeper is able to read image lists provided by EGI AppDB, parse their content and decide what cloud appliances should be added, updated or removed from managed cloud. During the addition and update cloudkeeper is able to download an appliance's image and convert it to the format supported by the managed cloud.
+
+Currently supported image formats are:
+* QCOW2
+* RAW
+* VMDK
+* OVA
+
+## How does cloudkeeper work?
+cloudkeeper communicates with cloud specific components via [gRPC](http://www.grpc.io/) communication framework to manage individual clouds. 
+
+Currently supported clouds:
+* [OpenNebula](https://opennebula.org/) - component [cloudkeeper-one](https://github.com/the-cloudkeeper-project/cloudkeeper-one)
+* [OpenStack](https://www.openstack.org/) - component [cloudkeeper-os](https://github.com/the-cloudkeeper-project/cloudkeeper-os) (under development)
+
+## Requirements
+* Ruby >= 2.2.0
+* Rubygems
+* qemu-img (image conversion utility)
+* NGINX (optional)
+
+## Installation
+
+### From RubyGems.org
+To install the most recent stable version
+```bash
+gem install cloudkeeper
+```
+
+### From source (dev)
+**Installation from source should never be your first choice! Especially, if you are not
+familiar with RVM, Bundler, Rake and other dev tools for Ruby!**
+
+**However, if you wish to contribute to our project, this is the right way to start.**
+
+To build and install the bleeding edge version from master
+
+```bash
+git clone git://github.com/the-cloudkeeper-project/cloudkeeper.git
+cd cloudkeeper
+gem install bundler
+bundle install
+bundle exec rake spec
+```
+
+## Configuration
+### Create a configuration file for cloudkeeper
+Configuration file can be read by cloudkeeper from these
+three locations:
+
+* `~/.cloudkeeper/cloudkeeper.yml`
+* `/etc/cloudkeeper/cloudkeeper.yml`
+* `PATH_TO_GEM_DIR/config/cloudkeeper.yml`
+
+The default configuration file can be found at the last location
+`PATH_TO_GEM_DIR/config/cloudkeeper.yml`.
+
+## Usage
+cloudkeeper is run with executable `cloudkeeper`. For further assistance run `cloudkeeper help sync`:
+```bash
+$ cloudkeeper help sync
+
+Usage:
+  cloudkeeper sync --backend-endpoint=BACKEND-ENDPOINT --formats=one two three --image-dir=IMAGE-DIR --image-lists=one two three --qemu-img-binary=QEMU-IMG-BINARY
+
+Options:
+  --image-lists=one two three                      # List of image lists to sync against
+  [--ca-dir=CA-DIR]                                # CA directory
+                                                   # Default: /etc/grid-security/certificates/
+  [--authentication], [--no-authentication]        # Client <-> server authentication
+  [--certificate=CERTIFICATE]                      # Core's host certificate
+                                                   # Default: /etc/grid-security/hostcert.pem
+  [--key=KEY]                                      # Core's host key
+                                                   # Default: /etc/grid-security/hostkey.pem
+  --image-dir=IMAGE-DIR                            # Directory to store images to
+                                                   # Default: /var/spool/cloudkeeper/images/
+  --qemu-img-binary=QEMU-IMG-BINARY                # Path to qemu-img binary (image conversion)
+                                                   # Default: /usr/bin/qemu-img
+  [--nginx-binary=NGINX-BINARY]                    # Path to nginx binary (HTTP server)
+                                                   # Default: /usr/bin/nginx
+  [--remote-mode], [--no-remote-mode]              # Remote mode starts HTTP server (NGINX) and serves images to backend via HTTP
+  [--nginx-error-log-file=NGINX-ERROR-LOG-FILE]    # NGINX error log file
+                                                   # Default: /var/log/cloudkeeper/nginx-error.log
+  [--nginx-access-log-file=NGINX-ACCESS-LOG-FILE]  # NGINX access log file
+                                                   # Default: /var/log/cloudkeeper/nginx-access.log
+  [--nginx-pid-file=NGINX-PID-FILE]                # NGINX pid file
+                                                   # Default: /var/run/cloudkeeper/nginx.pid
+  [--nginx-ip-address=NGINX-IP-ADDRESS]            # IP address NGINX can listen on
+                                                   # Default: 127.0.0.1
+  [--nginx-min-port=N]                             # Minimal port NGINX can listen on
+                                                   # Default: 7300
+  [--nginx-max-port=N]                             # Maximal port NGINX can listen on
+                                                   # Default: 7400
+  --backend-endpoint=BACKEND-ENDPOINT              # Backend's gRPC endpoint
+                                                   # Default: 127.0.0.1:50051
+  [--backend-certificate=BACKEND-CERTIFICATE]      # Backend's certificate
+                                                   # Default: /etc/grid-security/backendcert.pem
+  --formats=one two three                          # List of acceptable formats images can be converted to
+                                                   # Default: ["qcow2"]
+  --logging-level=LOGGING-LEVEL
+                                                   # Default: ERROR
+                                                   # Possible values: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
+  [--logging-file=LOGGING-FILE]                    # File to write logs to
+                                                   # Default: /var/log/cloudkeeper/cloudkeeper.log
+  [--debug], [--no-debug]                          # Runs cloudkeeper in debug mode
+```
+
+## Contributing
+1. Fork it ( https://github.com/the-cloudkeeper-project/cloudkeeper/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,17 @@
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+require 'bundler/gem_tasks'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: :spec
+
+desc 'Run acceptance tests (RSpec + Rubocop)'
+task test: 'acceptance'
+
+desc 'Run acceptance tests (RSpec + Rubocop)'
+task :acceptance do |_t|
+  Rake::Task['spec'].invoke
+  Rake::Task['rubocop'].invoke
+end

data/bin/cloudkeeper

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'cloudkeeper_grpc'
+require 'cloudkeeper'
+
+Cloudkeeper::CLI.start(ARGV)

data/cloudkeeper.gemspec

@@ -0,0 +1,45 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cloudkeeper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cloudkeeper'
+  spec.version       = Cloudkeeper::VERSION
+  spec.authors       = ['Michal Kimle']
+  spec.email         = ['kimle.michal@gmail.com']
+
+  spec.summary       = 'Synchronize cloud appliances between AppDB and cloud platforms'
+  spec.description   = 'Synchronize cloud appliances between AppDB and cloud platforms'
+  spec.homepage      = 'https://github.com/Misenko/cloudkeeper'
+  spec.license       = 'Apache License, Version 2.0'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 11.2'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rubocop', '~> 0.42'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.7'
+  spec.add_development_dependency 'rspec-collection_matchers', '~> 1.1'
+  spec.add_development_dependency 'simplecov', '~> 0.12'
+  spec.add_development_dependency 'pry', '~> 0.10'
+  spec.add_development_dependency 'vcr', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 2.1'
+  spec.add_development_dependency 'diffy', '~> 3.1'
+  spec.add_development_dependency 'grpc-tools', '~> 1.0'
+
+  spec.add_runtime_dependency 'thor', '~> 0.19'
+  spec.add_runtime_dependency 'yell', '~> 2.0'
+  spec.add_runtime_dependency 'mixlib-shellout', '~> 2.2'
+  spec.add_runtime_dependency 'grpc', '~> 1.0'
+  spec.add_runtime_dependency 'settingslogic', '~> 2.0'
+  spec.add_runtime_dependency 'zaru', '~> 0.1'
+  spec.add_runtime_dependency 'activesupport', '~> 4.0'
+  spec.add_runtime_dependency 'tilt', '~> 2.0'
+  spec.add_runtime_dependency 'faraday', '~> 0.11'
+
+  spec.required_ruby_version = '>= 2.2.0'
+end

data/config/cloudkeeper.yml

@@ -0,0 +1,27 @@
+cloudkeeper:
+  image-lists: # List of image lists to sync against
+  ca-dir: /etc/grid-security/certificates/ # CA directory
+  authentication: false # core (client) <-> backend (server) authentication (certificate, key and backend-certificate options)
+  certificate: /etc/grid-security/hostcert.pem # Core's host certificate
+  key: /etc/grid-security/hostkey.pem # Core's host key
+  image-dir: /var/spool/cloudkeeper/images/ # Directory to store images to
+  binaries:
+    qemu-img: /usr/bin/qemu-img # qemu-img binary (image conversion) location
+    nginx: /usr/bin/nginx # nginx binary (HTTP server) location
+  remote-mode: false # Remote mode starts HTTP server (NGINX) and serves images to backend via HTTP
+  nginx:
+    error-log-file: /var/log/cloudkeeper/nginx-error.log # File for NGINX error log
+    access-log-file: /var/log/cloudkeeper/nginx-access.log # File for NGINX access log
+    pid-file: /var/run/cloudkeeper/nginx.pid # NGINX pid file
+    ip-address: 127.0.0.1 # IP address NGINX can listen on
+    min-port: 7300 # Minimal port NGINX can listen on
+    max-port: 7400 # Maximal port NGINX can listen on
+  backend:
+    endpoint: 127.0.0.1:50051 # Backend's gRPC endpoint
+    certificate: /etc/grid-security/backendcert.pem # Backend's certificate
+  formats: # List of acceptable formats images can be converted to
+    - qcow2
+  logging:
+    level: ERROR # Logging level
+    file: /var/log/cloudkeeper/cloudkeeper.log # File to write log to. To turn off file logging leave this field empty.
+  debug: false # Debug mode

data/lib/cloudkeeper.rb

@@ -0,0 +1,16 @@
+require 'active_support/all'
+require 'mixlib/shellout'
+
+module Cloudkeeper
+  autoload :Settings, 'cloudkeeper/settings'
+  autoload :CLI, 'cloudkeeper/cli'
+  autoload :Utils, 'cloudkeeper/utils'
+  autoload :Nginx, 'cloudkeeper/nginx'
+  autoload :CommandExecutioner, 'cloudkeeper/command_executioner'
+  autoload :Entities, 'cloudkeeper/entities'
+  autoload :Managers, 'cloudkeeper/managers'
+  autoload :BackendConnector, 'cloudkeeper/backend_connector'
+  autoload :Errors, 'cloudkeeper/errors'
+end
+
+require 'cloudkeeper/version'

data/lib/cloudkeeper/backend_connector.rb

@@ -0,0 +1,123 @@
+module Cloudkeeper
+  class BackendConnector
+    include Cloudkeeper::Entities::Conversions
+
+    attr_reader :grpc_client, :nginx
+
+    def initialize
+      @grpc_client = CloudkeeperGrpc::Communicator::Stub.new(Cloudkeeper::Settings[:'backend-endpoint'], credentials)
+      @nginx = Cloudkeeper::Nginx::HttpServer.new
+    end
+
+    def pre_action
+      logger.debug "'pre_action' gRPC method call"
+      handle_errors grpc_client.pre_action(Google::Protobuf::Empty.new, return_op: true), raise_exception: true
+    end
+
+    def post_action
+      logger.debug "'post_action' gRPC method call"
+      handle_errors grpc_client.post_action(Google::Protobuf::Empty.new, return_op: true)
+    end
+
+    def add_appliance(appliance)
+      logger.debug "'add_appliance' gRPC method call"
+      manage_appliance appliance, :add_appliance
+    end
+
+    def update_appliance(appliance)
+      logger.debug "'update_appliance' gRPC method call"
+      manage_appliance appliance, :update_appliance
+    end
+
+    def remove_appliance(appliance)
+      logger.debug "'remove_appliance' gRPC method call"
+      manage_appliance appliance, :remove_appliance
+    end
+
+    def remove_image_list(image_list_identifier)
+      logger.debug "'remove_image_list' gRPC method call"
+      handle_errors grpc_client.remove_image_list(
+        CloudkeeperGrpc::ImageListIdentifier.new(image_list_identifier: image_list_identifier),
+        return_op: true
+      )
+    end
+
+    def image_lists
+      logger.debug "'image_lists' gRPC method call"
+      handle_errors(grpc_client.image_lists(Google::Protobuf::Empty.new, return_op: true)) do |response|
+        response.map(&:image_list_identifier)
+      end
+    end
+
+    def appliances(image_list_identifier)
+      logger.debug "'appliances' gRPC method call"
+      handle_errors(
+        grpc_client.appliances(
+          CloudkeeperGrpc::ImageListIdentifier.new(image_list_identifier: image_list_identifier),
+          return_op: true
+        )
+      ) do |response|
+        response.inject({}) do |acc, elem|
+          image = convert_image_proto(elem.image)
+          appliance = convert_appliance_proto elem, image
+          acc.merge appliance.identifier => appliance
+        end
+      end
+    end
+
+    private
+
+    def credentials
+      return :this_channel_is_insecure unless Cloudkeeper::Settings[:authentication]
+
+      GRPC::Core::ChannelCredentials.new(
+        File.read(Cloudkeeper::Settings[:'backend-certificate']),
+        File.read(Cloudkeeper::Settings[:key]),
+        File.read(Cloudkeeper::Settings[:certificate])
+      )
+    end
+
+    def handle_errors(operation, raise_exception: false)
+      return_value = operation.execute
+      return_value = yield(return_value) if block_given?
+      check_status operation.trailing_metadata, raise_exception: raise_exception
+
+      return_value
+    end
+
+    def check_status(metadata, raise_exception: false)
+      return if metadata[CloudkeeperGrpc::Constants::KEY_STATUS] == CloudkeeperGrpc::Constants::STATUS_SUCCESS
+
+      message = "#{metadata[CloudkeeperGrpc::Constants::KEY_STATUS]}: #{metadata[CloudkeeperGrpc::Constants::KEY_MESSAGE]}"
+      logger.error "Backend error: #{message}"
+      raise Cloudkeeper::Errors::BackendError, message if raise_exception
+    end
+
+    def set_remote_data(image_proto, access_data)
+      image_proto.mode = :REMOTE
+      image_proto.location = access_data[:url]
+      image_proto.username = access_data[:username]
+      image_proto.password = access_data[:password]
+    end
+
+    def prepare_image_proto(image)
+      image ? convert_image(image) : nil
+    end
+
+    def manage_appliance(appliance, call)
+      image = appliance.image
+      image_proto = prepare_image_proto image
+
+      if Cloudkeeper::Settings[:'remote-mode'] && image
+        nginx.start image_proto.location
+        set_remote_data image_proto, nginx.access_data
+      end
+
+      handle_errors grpc_client.send(call, convert_appliance(appliance, image_proto), return_op: true)
+
+      nginx.stop if Cloudkeeper::Settings[:'remote-mode'] && image
+    rescue Cloudkeeper::Errors::NginxError, Cloudkeeper::Errors::Image::Format::NoRequiredFormatAvailableError => ex
+      raise Cloudkeeper::Errors::Appliance::PropagationError, ex
+    end
+  end
+end