migratio 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72782d04daa9369d64692d88cbaf87a99c34f142
-  data.tar.gz: 87bb9fc2f18780825e8305eecb87775bd5fc3f92
+  metadata.gz: 5ae15f80e370bb4a7bec6df988bb24d8dc312a0d
+  data.tar.gz: 38d817bc87cf9706faaa1f7b62ccd797d699fb48
 SHA512:
-  metadata.gz: 79d7a6254b2c287e47bd0483affa51dba629a8e86ada70a20aa89db9b59766acd89ab633baeacc458e4e910a2318b41a802fe16324993f1dce47c6f45c00dcec
-  data.tar.gz: 991a54d76b02999d04f30a0f6372fe08043189cca98362a40a3bf653627141c3297b865f84f77258e34da2c0836f397a7ae20278b5f5f835596c107df85ab4a4
+  metadata.gz: 83d55980b32b9626ddfa8eba8235996fd86842ee69d99bd5375e9a1c3a69321999a26585b5868d6f8345b11edef4274e727127a476e5fcc77c316a19fa92894c
+  data.tar.gz: 4d9e480ae84e92c5c86d37519208f3a35d26e1109f619204cd29516c93cac0cd1b06da754bfefac605a90fa0d2ac434a53806f811d8fb2823d9e2415f2ada67a

data/.gitignore

@@ -1,24 +1,11 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-log/
-tmp
+*.db
+*.log
 config.yml
+
+.idea/
 *.ipr
 *.iws
 *.iml
-.idea/
-nbproject/
+
+/nbproject/private/
+/nbproject/

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in migration-worker.gemspec
+# Specify your gem's dependencies in migratio.gemspec
 gemspec

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    migratio (0.0.2)
+      sidekiq
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    connection_pool (2.1.1)
+    hitimes (1.2.2)
+    json (1.8.2)
+    rake (10.4.2)
+    redis (3.2.0)
+    redis-namespace (1.5.1)
+      redis (~> 3.0, >= 3.0.4)
+    sidekiq (3.3.0)
+      celluloid (>= 0.16.0)
+      connection_pool (>= 2.0.0)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    timers (4.0.1)
+      hitimes
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  migratio!
+  rake

data/README.md

@@ -9,6 +9,9 @@ This gem is used to perform virtual machine template migration between sites.
 - Linux (tested on Ubuntu)
 - Ruby 2.0+
 - Redis (can be installed on separate server)
+- OpenStack (with `nova` commands available for user who runs `migratio`)
+- Amazon EC2 CLI Tools (with `ec2-*` commands available for user who runs `migratio`, needs: Java Runtime Environment)
+- AWS CLI Tools (with `aws` command, needs: python and pip)
 
 ## Packages / Dependencies
 
@@ -21,7 +24,7 @@ Install additional packages (as root):
 
     aptitude install g++ make autoconf bison build-essential libssl-dev libyaml-dev libreadline6 libreadline6-dev zlib1g zlib1g-dev
 
-Install ``ruby`` and ``bundler`` (as root):
+Install `ruby` and `bundler` (as root):
 
     mkdir /tmp/ruby
     pushd /tmp/ruby
@@ -36,14 +39,10 @@ Install ``ruby`` and ``bundler`` (as root):
 
 Install this software (as non-root):
 
-    git clone https://github.com/paoolo/migratio.git /home/atmosphere/migratio
+    git clone https://github.com/dice-cyfronet/migratio.git /home/atmosphere/migratio
     cd /home/atmosphere/migratio
     cp /home/atmosphere/migratio/config.yml.example /home/atmosphere/migratio/config.yml
 
-Edit config file, set proper ``redis_url`` and name first row in queues with proper, unique name:
-
-    nano /home/atmosphere/migratio/config.yml
-
 Install gems:
 
     pushd /home/atmosphere/migratio
@@ -89,12 +88,12 @@ Install upstart scripts (as non-root, inside migratio directory):
     mkdir -p /home/atmosphere/.init
     cp -i /home/atmosphere/migratio/support/upstart/*.conf /home/atmosphere/.init/
 
-Set proper directory for ``migratio/`` and ``migratio/log/``:
+Set proper directory for `migratio/` and `migratio/log/`:
 
-    nano /home/atmosphere/.init/migration.conf
-    nano /home/atmosphere/.init/migratio-1.conf
+    nano /home/atmosphere/.init/migratio.conf
+    nano /home/atmosphere/.init/migratio-worker-1.conf
 
-Update profile files (eg. ``.bash_profile``):
+Update profile files (eg. `.bash_profile`):
 
     cat >> /home/atmosphere/.bash_profile <<EOL
     if [ ! -f /var/run/user/\$(id -u)/upstart/sessions/*.session ]
@@ -109,21 +108,81 @@ Update profile files (eg. ``.bash_profile``):
     EOL
     # you need to re-login to apply changes in /home/atmosphere/.bash_profile
 
+### Amazon EC2 CLI Tools
+
+Download Amazon EC2 CLI Tools:
+
+    wget http://s3.amazonaws.com/ec2-downloads/ec2-api-tools.zip
+
+Copy all executable files (unix scripts) from `bin/` to `/usr/bin`. Copy all libraries (jars) from `lib/` to `/usr/lib`.
+ 
+Add to `/etc/environment` following lines:
+
+    EC2_HOME=/usr
+    JAVA_HOME=/usr/lib/jvm/default-java
+
+### AWS CLI Tools
+
+Install `awscli` as `root`
+
+    pip install awscli
+
+Configure `awscli` for user who run `migratio`
+
+    aws configure
+
+## Configuration
+
+Edit config file. Set proper `redis_url` and `name`:
+
+    nano /home/atmosphere/migratio/config.yml
+
+`name` must be identical with ComputeSite `site_id` property. Eg. for ComputeSite with `name` `Local` and `site_id` `local`, `name` used in configuration is `local`, not `Local`.
+
+Create file `~/.creds` with credentials used in OpenStack and Amazon. Eg.
+
+    export OS_TENANT_NAME=openstack_tenant
+    export OS_USERNAME=openstack_username_for_tenant
+    export OS_PASSWORD=openstack_password_for_tenant
+    export OS_AUTH_URL="http://127.0.0.1:5000/v2.0/" # default value for OpenStack installation
+    export IMAGES_DIR=/var/lib/glance/images # default value for OpenStack installation
+    export SOURCE_CS=atmosphere_local_compute_site_site_id_property
+    export AWS_ACCESS_KEY=aws_access_key
+    export AWS_SECRET_KEY=aws_secret_key
+
+User who runs `migratio` need to be assigned to `glance` group. User running `migratio` need to be allow in `visudo` to run `qemu-img`.
+
+    atmosphere ALL=(root) NOPASSWD: /usr/bin/qemu-img
+
+Create configuration files per Compute Site in directory `config/`. Configuration file name must be identical with ComputeSite `site_id` property. Use suffix `.conf`. See [config/](config/).
+
+For OpenStack:
+
+    export EXTERNAL_USER=external_username
+    export EXTERNAL_HOST=external_ip_or_hostname
+
+Create account `external_username` and allow login without password (using ssh authorized keys) for user which is running `migratio`. Allow `external_username` to use OpenStack on external compute site.
+
+For Amazon:
+
+    export AWS_REGION=eu-west-1 # or eu-central-1, us-east-1, us-west-1, etc.
+    export EC2_URL=https://ec2.eu-west-1.amazonaws.com
+
 ## Usage
 
-First time run (as non-root, inside ``migratio/`` directory):
+First time run (as non-root, inside `migratio/` directory):
 
     pushd /home/atmosphere/migratio
-    bundle exec ./bin/run
+    bundle exec ./bin/migratio-run
     popd
 
 Normal run (as non-root):
 
-    start migration
+    start migratio
 
 Stop (as non-root):
 
-    stop migration
+    stop migratio
 
 ## Contributing
 

data/bin/migratio-run

@@ -1,12 +1,12 @@
 #!/usr/bin/env ruby
 require 'rubygems'
 require 'sidekiq'
-require_relative '../lib/migration/worker'
+require_relative '../lib/migratio/worker'
 
-RUNNER_PATH = File.join(ROOT_PATH, 'lib', 'migration', 'runner.rb')
+RUNNER_PATH = File.join(ROOT_PATH, 'lib', 'migratio', 'runner.rb')
 
 def queue
-  "migration_#{Migration::Worker.config.name}"
+  "migration_#{Migratio::Worker.config.name}"
 end
 
 system "sidekiq -q #{queue} -c 1 -r #{RUNNER_PATH}"

data/config.yml.example

@@ -1,3 +1,3 @@
 name: own_name
 redis_url: redis://localhost:6379
-namespace: air
+namespace: atmosphere

data/config/amazon.conf

@@ -1 +1,2 @@
-export AWS_REGION=eu-west-1
+export AWS_REGION=eu-west-1
+export EC2_URL=https://ec2.eu-west-1.amazonaws.com

data/config/external_openstack.conf

@@ -0,0 +1,2 @@
+export EXTERNAL_USER=external_username
+export EXTERNAL_HOST=external_ip_or_hostname

data/lib/migratio/worker/openstack_amazon_migrator.rb

@@ -8,8 +8,8 @@ module Migratio
         dir = File.dirname(__FILE__)
         output = `#{dir}/../../../scripts/openstack2amazon-convert.sh "#{image_uuid}" "#{dir}/../../../config/#{compute_site}.conf"`
         Sidekiq::Client.push(
-          'queue' => 'migration_jobs',
-          'class' => 'UpdateMigrationJobStatusWorker',
+          'queue' => 'feedback',
+          'class' => 'Atmosphere::UpdateMigrationJobStatusWorker',
           'args' => [image_uuid, config.name, compute_site, output])
         if $?.exitstatus == 1
           return
@@ -17,8 +17,8 @@ module Migratio
 
         output = `#{dir}/../../../scripts/openstack2amazon-transfer.sh "#{image_uuid}" "#{dir}/../../../config/#{compute_site}.conf"`
         Sidekiq::Client.push(
-          'queue' => 'migration_jobs',
-          'class' => 'UpdateMigrationJobStatusWorker',
+          'queue' => 'feedback',
+          'class' => 'Atmosphere::UpdateMigrationJobStatusWorker',
           'args' => [image_uuid, config.name, compute_site, output])
         if $?.exitstatus == 1
           return
@@ -26,8 +26,8 @@ module Migratio
 
         output = `#{dir}/../../../scripts/openstack2amazon-import.sh "#{image_uuid}" "#{dir}/../../../config/#{compute_site}.conf"`
         Sidekiq::Client.push(
-          'queue' => 'migration_jobs',
-          'class' => 'UpdateMigrationJobStatusWorker',
+          'queue' => 'feedback',
+          'class' => 'Atmosphere::UpdateMigrationJobStatusWorker',
           'args' => [image_uuid, config.name, compute_site, output])
         if $?.exitstatus == 1
           return
@@ -35,8 +35,8 @@ module Migratio
 
         output = `#{dir}/../../../scripts/openstack2amazon-register.sh "#{image_uuid}" "#{dir}/../../../config/#{compute_site}.conf"`
         Sidekiq::Client.push(
-          'queue' => 'migration_jobs',
-          'class' => 'UpdateMigrationJobStatusWorker',
+          'queue' => 'feedback',
+          'class' => 'Atmosphere::UpdateMigrationJobStatusWorker',
           'args' => [image_uuid, config.name, compute_site, output])
       end
     end

data/lib/migratio/worker/openstack_openstack_migrator.rb

@@ -8,8 +8,8 @@ module Migratio
         dir = File.dirname(__FILE__)
         output = `#{dir}/../../../scripts/openstack2openstack-transfer.sh "#{image_uuid}" "#{dir}/../../../config/#{compute_site}.conf"`
         Sidekiq::Client.push(
-          'queue' => 'migration_jobs',
-          'class' => 'UpdateMigrationJobStatusWorker',
+          'queue' => 'feedback',
+          'class' => 'Atmosphere::UpdateMigrationJobStatusWorker',
           'args' => [image_uuid, config.name, compute_site, output])
         if $?.exitstatus == 1
           return
@@ -17,8 +17,8 @@ module Migratio
 
         output = `#{dir}/../../../scripts/openstack2openstack-register.sh "#{image_uuid}" "#{dir}/../../../config/#{compute_site}.conf"`
         Sidekiq::Client.push(
-          'queue' => 'migration_jobs',
-          'class' => 'UpdateMigrationJobStatusWorker',
+          'queue' => 'feedback',
+          'class' => 'Atmosphere::UpdateMigrationJobStatusWorker',
           'args' => [image_uuid, config.name, compute_site, output])
       end
     end

data/lib/migratio/worker/version.rb

@@ -1,5 +1,5 @@
 module Migratio
   module Worker
-    VERSION = "0.0.2"
+    VERSION = "0.0.3"
   end
 end

data/support/upstart/{migration-worker-1.conf → migratio-worker-1.conf}

@@ -1,9 +1,9 @@
-start on starting migration-worker
-stop on stopping migration-worker
+start on starting migratio-worker
+stop on stopping migratio-worker
 respawn
 
 setuid atmosphere
-chdir /home/atmosphere/migration-worker
+chdir /home/atmosphere/migratio
 
 script
 # this script runs in /bin/sh by default
@@ -26,6 +26,6 @@ exec /bin/bash <<EOT
   # rvm
   # source /home/apps/.rvm/scripts/rvm
 
-  exec bundle exec ./bin/run >> /home/atmosphere/migration-worker/log/migration-worker.log 2>&1
+  exec bundle exec ./bin/migratio-run >> /home/atmosphere/migratio/log/migratio.log 2>&1
 EOT
-end script
+end script

data/support/upstart/migratio-worker.conf

@@ -0,0 +1,2 @@
+start on starting migratio
+stop on stopping migratio

data/support/upstart/{migration.conf → migratio.conf}

@@ -1,6 +1,6 @@
 pre-start script
-  mkdir -p /home/atmosphere/migration-worker/log
-  chown -R atmosphere /home/atmosphere/migration-worker/log
+  mkdir -p /home/atmosphere/migratio/log
+  chown -R atmosphere /home/atmosphere/migratio/log
 end script
 
 start on (started network-interface
@@ -9,4 +9,4 @@ start on (started network-interface
 
 stop on (stopping network-interface
          or stopping network-manager
-         or stopping networking)
+         or stopping networking)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: migratio
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Paweł Suder
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-13 00:00:00.000000000 Z
+date: 2015-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -56,23 +56,20 @@ description: Migration worker
 email:
 - pawel@suder.info
 executables:
-- migratio-client
 - migratio-run
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/migratio-client
 - bin/migratio-run
 - config.yml.example
 - config/amazon.conf
-- config/cyfronet-folsom.conf
-- config/cyfronet-icehouse.conf
-- config/vienna.conf
+- config/external_openstack.conf
 - lib/migratio/config.rb
 - lib/migratio/runner.rb
 - lib/migratio/worker.rb
@@ -89,9 +86,9 @@ files:
 - scripts/openstack2openstack-register.sh
 - scripts/openstack2openstack-transfer.sh
 - scripts/openstack2openstack.sh
-- support/upstart/migration-worker-1.conf
-- support/upstart/migration-worker.conf
-- support/upstart/migration.conf
+- support/upstart/migratio-worker-1.conf
+- support/upstart/migratio-worker.conf
+- support/upstart/migratio.conf
 homepage: https://github.com/dice-cyfronet/migratio
 licenses:
 - MIT

data/bin/migratio-client

@@ -1,15 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'sidekiq'
-
-require_relative '../lib/migration/worker/openstack_amazon_migrator'
-require_relative '../lib/migration/worker/openstack_openstack_migrator'
-require_relative '../lib/migration/config'
-require_relative '../lib/migration/worker'
-
-Sidekiq.configure_client do |c|
-  c.redis = { :namespace => Migration::Worker.config.namespace, :url => Migration::Worker.config.redis_url, queue: Migration::Worker.config.queue }
-end
-
-Sidekiq::Client.push('queue' => 'cyfronet', 'class' => Migration::Worker::AmazonExporter, 'args' => [])
-Sidekiq::Client.push('queue' => 'cyfronet', 'class' => Migration::Worker::OpenstackExporter, 'args' => [])

data/config/cyfronet-folsom.conf

@@ -1,2 +0,0 @@
-export EXTERNAL_USER=atmosphere
-export EXTERNAL_HOST=10.100.0.24

data/config/cyfronet-icehouse.conf

@@ -1,2 +0,0 @@
-export EXTERNAL_USER=atmosphere
-export EXTERNAL_HOST=10.100.0.4

data/config/vienna.conf

@@ -1,2 +0,0 @@
-export EXTERNAL_USER=atmosphere
-export EXTERNAL_HOST=10.0.0.1

data/support/upstart/migration-worker.conf

@@ -1,2 +0,0 @@
-start on starting migration
-stop on stopping migration