cloudflock 0.4.4 → 0.5.0

data/bin/cloudflock-files

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+$0 = 'cloudflock-files'
+
+require 'cloudflock/interface/cli/app/files'
+CLI = CloudFlock::Interface::CLI::Console
+Opts = CloudFlock::Interface::CLI::Opts
+
+# Trap C-c to kill the application in a friendly manner
+trap 'INT' do
+  puts "\nCaught SIGINT, exiting..."
+  exit 1
+end
+
+options = Opts.parse
+options[:config] = Opts.parse_config_file(options[:config_file])
+
+CloudFlock::Interface::CLI::App::Files.new(options)

data/lib/cloudflock/interface/cli/app/files.rb

@@ -0,0 +1,179 @@
+require 'cloudflock'
+require 'cloudflock/interface/cli'
+require 'cloudflock/remote/files'
+require 'tempfile'
+require 'thread'
+require 'fog'
+
+# Public: The Files app provides the interface to perform migrations of
+# File/Object storage (e.g. Amazon S3, Local files and Rackspace Cloud Files).  
+class CloudFlock::Interface::CLI::App::Files
+  CLI = CloudFlock::Interface::CLI::Console
+
+  DOWNLOAD_THREAD_COUNT = 4
+  UPLOAD_THREAD_COUNT   = 4
+
+  # Public: Begin Files migration on the command line
+  #
+  # opts - Hash containing options mappings.
+  def initialize(opts)
+    @options = opts
+    @download_finished = false
+    @download_mutex = Mutex.new
+    @upload_mutex = Mutex.new
+    @download_list = []
+    @upload_list = []
+
+    source_store = define_store("source")
+    destination_store = define_store("destination")
+
+    @source_container = define_container(source_store, "source")
+    @destination_container = define_container(destination_store, "destination",
+                                             true)
+
+    if perform_migration
+      puts "#{CLI.bold}#{CLI.blue}*** Migration complete#{CLI.reset}\a"
+    else
+      puts "#{CLI.bold}#{CLI.red}*** Migration failed#{CLI.reset}\a"
+    end
+  rescue Excon::Errors::Unauthorized => err
+    puts "A provider has returned an Unauthorized error."
+    puts err.inspect if @options[:verbose]
+    exit 1
+  end
+
+  # Internal: Migrate objects from the source store to the destination store.
+  #
+  # Returns a boolean value corresponding to whether the migration has
+  # completed successfully.
+  def perform_migration
+    download_threads = []
+    upload_threads = []
+
+    @source_container.files.each { |f| @download_list.push(f) }
+
+    DOWNLOAD_THREAD_COUNT.times do
+      download_threads << download_thread
+    end
+    UPLOAD_THREAD_COUNT.times do
+      upload_threads << upload_thread
+    end
+
+    download_threads.each { |t| t.join }
+    @download_finished = true
+    upload_threads.each { |t| t.join }
+    true
+  rescue => e
+    if @options[:verbose]
+      puts "#{CLI.bold}#{CLI.red}*** Error ***#{CLI.reset}"
+      puts e.inspect
+      puts e.backtrace
+      puts
+    end
+    false
+  end
+
+  # Internal: Create a new Thread to download objects from the source
+  # container.
+  #
+  # Returns a Thread.
+  def download_thread
+    Thread.new do
+      file = nil
+      until @download_list.empty?
+        @download_mutex.synchronize do
+          file = @download_list.pop
+        end
+        next if file.nil?
+        # AWS stores directories as their own object
+        next if file.content_length == 0 && file.key =~ /\/$/
+
+        tmp = Tempfile.new(file.object_id.to_s)
+        @source_container.files.get(file.key) do |data, rem, cl|
+          tmp.syswrite(data)
+        end
+        tmp.flush
+        tmp.rewind
+        @upload_mutex.synchronize do
+          @upload_list.push(body: tmp, key: file.key)
+        end
+      end
+    end
+  end
+
+  # Internal: Create a new Thread to upload objects to the desination
+  # container.
+  #
+  # Returns a Thread.
+  def upload_thread
+    Thread.new do
+      file = nil
+      until @upload_list.empty? && @download_finished
+        sleep 0.1
+        @upload_mutex.synchronize do
+          file = @upload_list.pop
+        end
+        next if file.nil?
+        @destination_container.files.create(file)
+      end
+    end
+  end
+
+  # Internal: Ascertain the location for a data store.
+  #
+  # desc - String containing a description for the file store.
+  #
+  # Returns a Fog object pointing to the data store.
+  # Raises ArgumentError if desc isn't a String.
+  def define_store(desc)
+    unless desc.kind_of?(String)
+      raise ArgumentError, "String expected"
+    end
+    store = {}
+    store[:provider] = CLI.prompt("#{desc} provider (aws, local, rax)",
+                                  valid_answers: ["rax", "aws", "local"])
+    case store[:provider]
+    when 'rax'
+      store[:provider] = 'Rackspace'
+      store[:rackspace_username] = CLI.prompt("Rackspace username")
+      store[:rackspace_api_key] = CLI.prompt("Rackspace API key")
+    when 'aws'
+      store[:provider] = 'AWS'
+      store[:aws_access_key_id] = CLI.prompt("AWS Access Key ID")
+      store[:aws_secret_access_key] = CLI.prompt("AWS secret access key")
+    when 'local'
+      store[:local_root] = CLI.prompt("#{desc} location")
+    end
+
+    CloudFlock::Remote::Files.connect(store)
+  end
+
+  # Internal: Obtain the name of a container.
+  #
+  # store  - Fog object pointing to a Fog::Storage object.
+  # desc   - String containing a description for the container.
+  # create - Boolean value indicating whether to create the container.
+  #
+  # Returns a Fog object pointing to the container.
+  # Raises ArgumentError if store isn't a Fog::Storage object.
+  # Raises ArgumentError if desc isn't a String.
+  def define_container(store, desc, create=false)
+    unless store.class.to_s =~ /^Fog::Storage/
+      raise ArgumentError, "Fog Storage object expected"
+    end
+    unless desc.kind_of?(String)
+      raise ArgumentError, "String expected"
+    end
+
+    if create
+      container = CLI.prompt("#{desc} container name")
+      return store.directories.create(key: container)
+    else
+      puts "Available containers:"
+      puts store.directories.map(&:key)
+      container = CLI.prompt("#{desc} container name",
+                             valid_answers: store.directories.map(&:key))
+      return store.directories.select { |i| i.key == container }[0]
+    end
+  end
+end

data/lib/cloudflock/remote/files.rb

@@ -0,0 +1,28 @@
+require 'cloudflock'
+
+# Public: Provide an interface to instantiate Fog::Storage instances, perform
+# basic sanity checking for the local provider.
+module CloudFlock::Remote::Files extend self
+  # Public: Set up and verify a data source.
+  #
+  # target - Hash containing connection details per Fog requirements, a String
+  #          containing the path to a local directory.
+  #
+  # Returns an instance of a Files subclass.
+  def connect(target)
+    target = { provider: 'local', local_root: target } if target.is_a?(String)
+    raise ArgumentError, "String or Hash expected" unless target.is_a?(Hash)
+
+    if target[:provider].downcase == 'local'
+      target_parent = File.expand_path(target[:local_root], '..')
+      unless File.exists?(target_parent)
+        raise Errno::ENOENT, "#{target_parent} does not exist"
+      end
+      unless File.directory?(target_parent)
+        raise Errno::ENOENT, "#{target_parent} is not a directory"
+      end
+    end
+
+    Fog::Storage.new(target)
+  end
+end

data/lib/cloudflock/target/servers/data/exceptions/platform/centos.txt

@@ -0,0 +1,6 @@
+/etc/yum.repos.d/
+/usr/lib/yum-plugins
+/etc/yum.conf
+/etc/yum
+/etc/yum.repos.d
+/etc/sysconfig/iptables

data/lib/cloudflock/target/servers/data/exceptions/platform/redhat.txt

@@ -3,3 +3,4 @@
 /etc/yum.conf
 /etc/yum
 /etc/yum.repos.d
+/etc/sysconfig/iptables

data/lib/cloudflock/target/servers/migrate.rb

@@ -69,7 +69,7 @@ module CloudFlock::Target::Servers::Migrate extend self
     preserve_files = ["passwd", "shadow", "group"]
     path = "/mnt/migration_target/etc"
     preserve_files.each do |file|
-      copy_command = "[ -f #{path}/migration.#{file} ] || /bin/cp -an " +
+      copy_command = "[ -f #{path}/migration.#{file} ] || /bin/cp -a " +
                      "#{path}/#{file} #{path}/migration.#{file}"
       host.puts(copy_command)
       host.prompt

data/lib/cloudflock/version.rb

@@ -1,3 +1,3 @@
 module CloudFlock
-  VERSION = '0.4.4'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,55 +1,62 @@
 --- !ruby/object:Gem::Specification
 name: cloudflock
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
+  prerelease: 
 platform: ruby
 authors:
 - Chris Wuest
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-03 00:00:00.000000000 Z
+date: 2013-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.11.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.11.1
 - !ruby/object:Gem::Dependency
   name: expectr
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: cpe
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: CloudFlock is a library and toolchain focused on migration
@@ -65,17 +72,19 @@ files:
 - lib/cloudflock/error.rb
 - lib/cloudflock/interface/cli.rb
 - lib/cloudflock/interface/cli/app/common/servers.rb
+- lib/cloudflock/interface/cli/app/files.rb
 - lib/cloudflock/interface/cli/app/servers.rb
 - lib/cloudflock/interface/cli/app/servers/migrate.rb
 - lib/cloudflock/interface/cli/app/servers/profile.rb
 - lib/cloudflock/interface/cli/console.rb
 - lib/cloudflock/interface/cli/opts.rb
 - lib/cloudflock/interface/cli/opts/servers.rb
-- lib/cloudflock/patch/fog.rb
+- lib/cloudflock/remote/files.rb
 - lib/cloudflock/remote/ssh.rb
 - lib/cloudflock/target/servers.rb
 - lib/cloudflock/target/servers/data/exceptions/base.txt
 - lib/cloudflock/target/servers/data/exceptions/platform/amazon.txt
+- lib/cloudflock/target/servers/data/exceptions/platform/centos.txt
 - lib/cloudflock/target/servers/data/exceptions/platform/debian.txt
 - lib/cloudflock/target/servers/data/exceptions/platform/redhat.txt
 - lib/cloudflock/target/servers/data/exceptions/platform/suse.txt
@@ -90,32 +99,33 @@ files:
 - lib/cloudflock/target/servers/profile.rb
 - lib/cloudflock/version.rb
 - bin/cloudflock
+- bin/cloudflock-files
 - bin/cloudflock-profile
 - bin/cloudflock-servers
 - bin/cloudflock.default
 homepage: http://github.com/cwuest/cloudflock
 licenses:
 - Apache 2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Server migration automation
 test_files: []
-has_rdoc: 

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 8d1d382459026ce9c6cc5b6b2c340dde31b6b035
-  data.tar.gz: 829d46918880f162a85babcd15217808c56d468f
-SHA512:
-  metadata.gz: 6262061b988a8bd8a8a60722220942ce1ff8baa01a8bad37995ef38e91ed749b6f3c0132003e0db09a978e1844cadaef72fb43a8f1be38fd0080af2d5f2c3658
-  data.tar.gz: f57c2a93299ea7bea12413d6c72d8ccb7d5f00642c746d5b2f6eccb0e0863b2415b4c39d8a1b15b026e6f600d79f97085c4cab496dc8d5506b3603f9443276f8

data/lib/cloudflock/patch/fog.rb

@@ -1,113 +0,0 @@
-require 'fog'
-module Fog
-  module Compute
-    class RackspaceV2
-      class Server
-        # Place existing server into rescue mode, allowing for offline editing of configuration. The original server's disk is attached to a new instance of the same base image for a period of time to facilitate working within rescue mode.  The original server will be autom atically restored after 90 minutes.
-        # @return [Boolean] returns true if call to put server in rescue mode returns success
-        # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
-        # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
-        # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
-        # @raise [Fog::Rackspace::Errors::ServiceError]
-        # @note Rescue mode is only guaranteed to be active for 90 minutes.
-        # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/rescue_mode.html
-        # @see #unrescue
-        #
-        # * Status Transition:
-        #   * ACTIVE -> PREP_RESCUE -> RESCUE
-        def rescue
-          requires :identity
-          data = service.rescue_server(identity)
-          merge_attributes(data.body)
-          self.state = RESCUE
-          true
-        end
-
-        # Remove existing server from rescue mode.
-        # @return [Boolean] returns true if call to remove server from rescue mode returns success
-        # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
-        # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
-        # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
-        # @raise [Fog::Rackspace::Errors::ServiceError]
-        # @note Rescue mode is only guaranteed to be active for 90 minutes.
-        # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/exit_rescue_mode.html
-        # @see #rescue
-        #
-        # * Status Transition:
-        #   * RESCUE -> PREP_UNRESCUE -> ACTIVE
-        def unrescue
-          requires :identity
-          service.unrescue_server(identity)
-          self.state = ACTIVE
-          true
-        end
-      end
-
-      class Real
-        # Puts server into rescue mode
-        # @param [String] server_id id of server to rescue
-        # @return [Excon::Response] response
-        # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
-        # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
-        # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
-        # @raise [Fog::Rackspace::Errors::ServiceError]
-        # @note Rescue mode is only guaranteed to be active for 90 minutes.
-        # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/rescue_mode.html
-        #
-        # * Status Transition:
-        #   * ACTIVE -> PREP_RESCUE -> RESCUE
-        def rescue_server(server_id)
-          data = {
-            'rescue' => nil
-          }
-
-          request(
-            :body => Fog::JSON.encode(data),
-            :expects => [200],
-            :method => 'POST',
-            :path => "servers/#{server_id}/action"
-          )
-        end
-
-        # Take server out of rescue mode
-        # @param [String] server_id id of server
-        # @return [Excon::Response] response
-        # @raise [Fog::Rackspace::Errors::NotFound] - HTTP 404
-        # @raise [Fog::Rackspace::Errors::BadRequest] - HTTP 400
-        # @raise [Fog::Rackspace::Errors::InternalServerError] - HTTP 500
-        # @raise [Fog::Rackspace::Errors::ServiceError]
-        # @see http://docs.rackspace.com/servers/api/v2/cs-devguide/content/exit_rescue_mode.html
-        #
-        # * Status Transition:
-        #   * RESCUE -> PREP_UNRESCUE -> ACTIVE
-        def unrescue_server(server_id)
-          data = {
-            'unrescue' => nil
-          }
-
-          request(
-            :body => Fog::JSON.encode(data),
-            :expects => [202],
-            :method => 'POST',
-            :path => "servers/#{server_id}/action"
-          )
-        end
-      end
-
-      class Mock
-        def rescue_server(server_id)
-          server = self.data[:servers][server_id]
-          server["status"] = "PREP_RESCUE"
-          response(:status => 200)
-        end
-
-        def unrescue_server(server_id)
-          server = self.data[:servers][server_id]
-          server["status"] = "PREP_UNRESCUE"
-          response(:status => 202)
-        end
-      end
-    end
-  end
-end
-