cloudflock 0.6.1 → 0.7.0

data/lib/cloudflock/app/files-migrate.rb

@@ -0,0 +1,246 @@
+require 'thread'
+require 'tempfile'
+require 'cloudflock/remote/files'
+require 'cloudflock/app/common/rackspace'
+require 'cloudflock/app'
+
+module CloudFlock; module App
+  # Public: The FilesMigrate class provides the interface to perform migrations
+  # to and from Cloud Files containers, S3 buckets, and local file stores.
+  class FilesMigrate
+    include CloudFlock::App::Rackspace
+    include CloudFlock::Remote
+
+    # Default number of threads to be used to upload staged files.
+    UPLOAD_THREADS   = 20
+
+    # Default number of threads to be used to download files to staging area.
+    DOWNLOAD_THREADS = 20
+
+    # Public: Perform the steps necessary to migrate files from one file store
+    # to another.
+    def initialize
+      options      = parse_options
+      source_store = define_source
+      dest_store   = define_destination
+
+      UI.spinner('Migrating files') do
+        files_migrate(source_store, dest_store, options)
+      end
+    end
+
+    private
+
+    # Internal: Gather information for and connect to the source store.
+    #
+    # Returns a CloudFlock::Remote::Files object.
+    def define_source
+      define_api('Source')
+    end
+
+    # Internal: Gather information for and connect to the destination store.
+    #
+    # Returns a CloudFlock::Remote::Files object.
+    def define_destination
+      define_api('Destination', true)
+    end
+
+    # Internal: Obtain information needed to connect to a data store.
+    #
+    # desc   - Description of the data store for display purposes.
+    # create - Whether to create non-existing locations. (default: false)
+    #
+    # Returns a CloudFlock::Remote::Files object.
+    def define_api(desc, create = false)
+      store   = {}
+      query   = "#{desc} provider (rackspace, aws, local)"
+      answers = [/^(?:rackspace|aws|local)$/i]
+
+      provider = UI.prompt(query, valid_answers: answers)
+
+      case provider
+      when /rackspace/i
+        store[:provider] = 'Rackspace'
+        store[:rackspace_username] = UI.prompt('Rackspace username')
+        store[:rackspace_api_key] = UI.prompt('Rackspace API key')
+      when /aws/i
+        store[:provider] = 'AWS'
+        store[:aws_access_key_id] = UI.prompt('AWS Access Key ID')
+        store[:aws_secret_access_key] = UI.prompt('AWS secret access key')
+      when /local/i
+        store[:provider] = 'local'
+        store[:local_root] = UI.prompt("#{desc} location")
+        return CloudFlock::Remote::Files.new(store)
+      end
+
+      api = CloudFlock::Remote::Files.new(store)
+
+      options = api.directories.map do |dir|
+        { name: dir.key, files: dir.count.to_s }
+      end
+      valid = options.reduce([]) { |c,e| c << e[:name] }
+
+      puts UI.build_grid(options, name: "Directory name", files: "File count")
+      if create
+        selected = UI.prompt("#{desc} directory")
+        unless api.directories.select { |dir| dir.key == selected }.any?
+          api.directories.create(key: selected)
+        end
+      else
+        selected = UI.prompt("#{desc} directory", valid_answers: valid)
+      end
+      api.directory = selected
+
+      api
+    end
+
+    # Internal: Set up queue and Mutexes, create threads to manage the transfer
+    # of files from source to destination.
+    #
+    # source_store - CloudFlock::Remote::Files object set up to pull files from
+    #                a source directory.
+    # dest_store   - CloudFlock::Remote::Files object set up to create files as
+    #                they are uploaded.
+    # options      - Hash optionally containing overrides for the number of
+    #                upload and download threads to use for transfer
+    #                concurrency. (default: {})
+    #                :upload_threads   - Number of upload threads to use.
+    #                                    Overrides UPLOAD_THREADS constant.
+    #                :download_threads - Number of download threads to use.
+    #                                    Overrides DOWNLOAD_THREADS constant.
+    #
+    # Returns nothing.
+    def files_migrate(source_store, dest_store, options = {})
+      file_queue       = []
+      mutexes          = { queue: Mutex.new, finished: Mutex.new }
+      up_threads       = options[:upload_threads] || UPLOAD_THREADS
+      down_threads     = options[:download_threads] || DOWNLOAD_THREADS
+
+      source = Thread.new do
+        manage_source(source_store, file_queue, mutexes, up_threads)
+      end
+
+      destination = Thread.new do
+        manage_destination(dest_store, file_queue, mutexes, down_threads)
+      end
+
+      [source, destination].each(&:join)
+    end
+
+    # Internal: Create and observe threads which download files from a
+    # non-local file store.  If the files exist locally, simply generate a list
+    # of the files in queue.
+    #
+    # source_store - CloudFlock::Remote::Files object set up to pull files from
+    #                a source directory.
+    # file_queue   - Array in which details regarding files to be transferred
+    #                will be stored.
+    # mutexes      - Hash containing two Mutexes:
+    #                :queue    - Coordinates access to file_queue.
+    #                :finished - Locked immediately, only unlocked once all
+    #                            files are queued for transfer.
+    # thread_count - Hash optionally containing overrides for the number of
+    #                upload and download threads to use for transfer
+    #                concurrency. (default: {})
+    #                :upload_threads   - Number of upload threads to use.
+    #                                    Overrides UPLOAD_THREADS constant.
+    #                :download_threads - Number of download threads to use.
+    #                                    Overrides DOWNLOAD_THREADS constant.
+    #
+    # Returns nothing.
+    def manage_source(source_store, file_queue, mutexes, thread_count)
+      mutexes[:finished].lock
+
+      if source_store.local?
+        source_store.each_file do |file|
+          mutexes[:queue].synchronize do
+            file_queue << { path: "#{source_store.prefix}/#{file.key}",
+                            name: file.key, temp: false }
+          end
+        end
+      else
+        source_threads  = []
+        file_list_mutex = Mutex.new
+        file_list       = source_store.file_list
+
+        thread_count.times do
+          source_threads << Thread.new do
+            while file = file_list_mutex.synchronize { file_list.pop } do
+              temp = Tempfile.new(file.gsub(/\//, ''))
+              temp.write(source_store.get_file(file))
+              temp.close
+              mutexes[:queue].synchronize do
+                file_queue << { path: temp.path, name: file, temp: true }
+              end
+            end
+          end
+        end
+
+        source_threads.each(&:join)
+      end
+
+      mutexes[:finished].unlock
+    end
+
+    # Internal: Create and observe threads which download files from a
+    # non-local file store.  If the files exist locally, simply generate a list
+    # of the files in queue.
+    #
+    # dest_store   - CloudFlock::Remote::Files object set up to upload files to
+    #                a destination directory.
+    # file_queue   - Array from which to retrieve details regarding files to be
+    #                transferred.
+    # mutexes      - Hash containing two Mutexes:
+    #                :queue    - Coordinates access to file_queue.
+    #                :finished - Locked immediately, only unlocked once all
+    #                            files are queued for transfer.
+    # thread_count - Hash optionally containing overrides for the number of
+    #                upload and download threads to use for transfer
+    #                concurrency. (default: {})
+    #                :upload_threads   - Number of upload threads to use.
+    #                                    Overrides UPLOAD_THREADS constant.
+    #                :download_threads - Number of download threads to use.
+    #                                    Overrides DOWNLOAD_THREADS constant.
+    #
+    # Returns nothing.
+    def manage_destination(dest_store, file_queue, mutexes, thread_count)
+      dest_threads = []
+
+      thread_count.times do
+        dest_threads << Thread.new do
+          while mutexes[:finished].locked?
+            while file = mutexes[:queue].synchronize { file_queue.pop }
+              content = File.read(file[:path])
+              dest_store.create(key: file[:name], body: content)
+              File.unlink(file[:path]) if file[:temp]
+            end
+          end
+        end
+      end
+
+      dest_threads.each(&:join)
+    end
+
+    # Internal: Set up an OptionParser object to recognize options specific to
+    # profiling a remote host.
+    #
+    # Returns nothing.
+    def parse_options
+      options = {}
+
+      CloudFlock::App.parse_options(options) do |opts|
+        opts.separator 'Migrate files between file stores'
+        opts.separator ''
+
+        opts.on('-u', '--upload-threads THREADS',
+                'Number of upload threads to use (default 20)') do |threads|
+          options[:upload_threads] = threads.to_i if threads.to_i > 0
+        end
+        opts.on('-d', '--download-threads THREADS',
+                'Number of download threads to use (default 20)') do |threads|
+          options[:download_threads] = threads.to_i if threads.to_i > 0
+        end
+      end
+    end
+  end
+end; end

data/lib/cloudflock/app/server-migrate.rb

@@ -0,0 +1,327 @@
+require 'cloudflock/app/common/servers'
+require 'cloudflock/task/server-profile'
+require 'cloudflock/app'
+require 'tempfile'
+require 'fog'
+
+module CloudFlock; module App
+  # Public: The ServerMigrate class provides the interface to perform one-shot
+  # migrations as a CLI application.
+  class ServerMigrate
+    include CloudFlock::App::Common
+    include CloudFlock::Remote
+
+    # Public: Perform the steps necessary to migrate a Unix host to a standing
+    # host or to a newly provisioned Rackspace Cloud server.
+    def initialize
+      options = parse_options
+
+      source_host = source_connect(options)
+      profile = fetch_profile(source_host)
+
+      puts generate_recommendation(profile)
+
+      dest_host = destination_connect(options, profile)
+      exclusions = build_exclusions(profile.cpe)
+      migrate_server(source_host, dest_host, exclusions)
+
+      source_host.logout!
+      cleanup_destination(dest_host, profile.cpe)
+      configure_ips(dest_host, profile)
+
+      puts UI.bold { UI.blue { "Migration complete to #{dest_host.hostname}"} }
+    rescue
+      puts UI.red { 'An unhandled error was encountered.  Details follow:' }
+      raise
+    end
+
+    private
+
+    # Internal: Profile a server in order to make accurate recommendations.
+    #
+    # source_ssh - SSH object connected to a Unix host.
+    #
+    # Returns a ServerProfile object.
+    def fetch_profile(source_ssh)
+      UI.spinner("Checking source host") do
+        CloudFlock::Task::ServerProfile.new(source_ssh)
+      end
+    end
+
+    # Internal: Collect information needed to connect to the source host for a
+    # migration.  Connect to the target host.
+    #
+    # options - Hash containing information to connect to an existing host.
+    #
+    # Returns an SSH object connected to the source host.
+    def source_connect(options)
+      source_host = define_source(options)
+      ssh_connect(source_host)
+    end
+
+    # Internal: Collect information needed to either connect to an existing
+    # host or provision a new one on Rackspace Cloud to be used as a target
+    # for migration.  Connect to the target host.
+    #
+    # options - Hash containing information to connect to an existing host.
+    # profile - ServerProfile for the source host.
+    #
+    # Returns an SSH object connected to the target host.
+    def destination_connect(options, profile)
+      if options[:resume]
+        dest_host = define_destination(options)
+      else
+        api = define_rackspace_api
+        managed = UI.prompt_yn('Managed account? (Y/N)', default_answer: 'N')
+        dest_host = create_cloud_instance(api, profile, managed)
+      end
+
+      ssh_connect(dest_host)
+    rescue Excon::Errors::Unauthorized
+      retry if UI.prompt_yn('Login failed.  Retry? (Y/N)', default_answer: 'Y')
+      exit
+    end
+
+    # Internal: Provision a new instance on the Rackspace cloud and return
+    # credentials once finished.
+    #
+    # api     - Hash containing credentials to interact with the Rackspace
+    #           Cloud API.
+    # profile - ServerProfile for the source host.
+    # managed - Whether the account is a Managed Cloud account (needed to know.
+    #           whether to wait for post-provisioning automation to finish)
+    #
+    # Returns a Hash containing credentials suitable for logging in via SSH.
+    def create_cloud_instance(api, profile, managed)
+      api = define_rackspace_cloudservers_region(api)
+      compute = Fog::Compute.new(api)
+      image = define_compute_image(compute, profile)
+      flavor = define_compute_flavor(compute, profile)
+      name = define_compute_name(profile)
+
+      compute_spec = { image_id: image, flavor_id: flavor, name: name }
+      provision_compute(compute, managed, compute_spec)
+    end
+
+    # Internal: Generate a recommendation based on the results of profiling a
+    # host.
+    #
+    # profile - ServerProfile for the source host.
+    #
+    # Returns a String.
+    def generate_recommendation(profile)
+      os  = profile_os_string(profile)
+      ram = profile_ram_string(profile)
+      hdd = profile_hdd_string(profile)
+
+      "OS:  " + UI.bold { os } + "\n" +
+      "RAM: " + UI.bold { ram } + "\n" +
+      "HDD: " + UI.bold { hdd } + "\n" +
+      UI.red  { UI.bold { profile.warnings.join("\n") } }
+    end
+
+    # Internal: Build exclusions list based on a host's CPE.
+    #
+    # cpe - CPE object describing a given host.
+    #
+    # Returns a String
+    def build_exclusions(cpe)
+      exclusions = Exclusions.new(cpe)
+      edit = UI.prompt_yn('Edit exclusions list? (Y/N)', default_answer: 'N')
+      exclusions = edit_exclusions(exclusions) if edit
+
+      exclusions.to_s
+    end
+
+    # Internal: Allow editing of the default exclusions for a given platform.
+    #
+    # exclusions - String containing exclusions.
+    #
+    # Returns a String.
+    def edit_exclusions(exclusions)
+      temp_file('exclusions', exclusions)
+    end
+
+    # Internal: Allow editing of a list of IPs.
+    #
+    # ips - Array containing Strings of IPs.
+    #
+    # Returns an Array containing Strings of IPs.
+    def edit_ip_list(ips)
+      temp_file('ips', ips.join("\n")).split(/\s+/)
+    end
+
+    # Internal: Allow editing of a list of target directories.
+    #
+    # dirs - Array containing Strings of paths.
+    #
+    # Returns an Array containing Strings of paths.
+    def edit_directory_list(dirs)
+      temp_file('directories', dirs.join("\n")).split(/\s+/)
+    end
+
+    # Internal: Generate a String describing a host's operating system.
+    #
+    # profile - ServerProfile for the source host.
+    #
+    # Returns a String.
+    def profile_os_string(profile)
+      os = profile.select_entries(/System/, 'OS')
+      os += profile.select_entries(/System/, 'Arch')
+      os.map(&:capitalize).join(' ')
+    end
+
+    # Internal: Generate a String describing a host's memory usage.
+    #
+    # profile - ServerProfile for the source host.
+    #
+    # Returns a String.
+    def profile_ram_string(profile)
+      profile.select_entries(/Memory/, /Used RAM/).join.strip
+    end
+
+    # Internal: Generate a String describing a host's disk usage.
+    #
+    # profile - ServerProfile for the source host.
+    #
+    # Returns a String.
+    def profile_hdd_string(profile)
+      profile.select_entries(/Storage/, /Usage/).join(' ').strip
+    end
+
+    # Internal: Set up a temporary file, open it for editing locally, and read
+    # it back in after finished.
+    #
+    # name    - Name to append to the temporary file's path.
+    # content - Content with which the temporary file should be pre-populated.
+    #
+    # BUG: Works only on POSIX-compliant hosts; needs work to support Windows.
+    #
+    # Returns a String.
+    def temp_file(name, content)
+      editor = File.exists?('/usr/bin/editor') ? '/usr/bin/editor' : 'vi'
+
+      temp = Tempfile.new("cloudflock_#{name}")
+      temp.write(content)
+      temp.close
+
+      system("#{editor} #{temp.path}")
+      temp.open
+      result = temp.read
+      temp.close
+      temp.unlink
+
+      result
+    end
+
+    # Internal: Set up an OptionParser object to recognize options specific to
+    # profiling a remote host.
+    #
+    # Returns nothing.
+    def parse_options
+      options = {}
+
+      CloudFlock::App.parse_options(options) do |opts|
+        opts.separator 'Perform host-level migration'
+        opts.separator ''
+        opts.separator 'Options for source definition:'
+
+        begin # Source options
+          opts.on('-h', '--src-host HOST', 'Address for source host') do |host|
+            options[:hostname] = host
+          end
+
+          opts.on('-p', '--src-port PORT',
+                  'Source SSH port for source host') do |port|
+            options[:port] = port
+          end
+
+          opts.on('-u', '--src-user USER',
+                  'Username for source host') do |user|
+            options[:username] = user
+          end
+
+          opts.on('-a', '--src-password [PASSWORD]',
+                  'Password for source host login') do |pass|
+            options[:password] = pass
+          end
+
+          opts.on('-s', '--src-sudo', 'Use sudo to gain root on source host') do
+            options[:sudo] = true
+          end
+
+          opts.on('-n', '--src-no-sudo', 'Use su to gain root on source host') do
+            options[:sudo] = false
+          end
+
+          opts.on('-r', '--src-root-pass PASS',
+                  'Password for root user on the source host') do |root|
+            options[:root_password] = root
+          end
+
+          opts.on('-i', '--src-identity IDENTITY',
+                  'SSH identity to use for the source host') do |key|
+            options[:ssh_key] = key
+          end
+        end
+
+        opts.separator ''
+        opts.separator 'Options for destination (if not using automation):'
+
+        begin # Destination options
+          opts.on('-H', '--dest-host HOST',
+                  'Address for destination host') do |host|
+            options[:dest_hostname] = host
+          end
+
+          opts.on('-P', '--dest-port PORT',
+                  'Source SSH port for destination host') do |port|
+            options[:dest_port] = port
+          end
+
+          opts.on('-U', '--dest-user USER',
+                  'Username for destination host') do |user|
+            options[:dest_username] = user
+          end
+
+          opts.on('-A', '--dest-password [PASSWORD]',
+                  'Password for destination host login') do |pass|
+            options[:dest_password] = pass
+          end
+
+          opts.on('-S', '--dest-sudo', 'Use sudo to gain root on destination host') do
+            options[:dest_sudo] = true
+          end
+
+          opts.on('-N', '--dest-no-sudo', 'Use su to gain root on destination host') do
+            options[:dest_sudo] = false
+          end
+
+          opts.on('-R', '--dest-root-pass PASS',
+                  'Password for root user on the destination host') do |root|
+            options[:dest_root_password] = root
+          end
+
+          opts.on('-I', '--dest-identity IDENTITY',
+                  'SSH identity to use for the destination host') do |key|
+            options[:dest_ssh_key] = key
+          end
+        end
+
+        opts.separator ''
+        opts.separator 'Operation options:'
+
+        begin # Operation options
+          opts.on('--resume', '--pre-provisioned',
+                  'Migrate over standing host ("resume" mode)') do
+            options[:resume] = true
+          end
+          opts.on('--echo-passwords',
+                  'Echo entered passwords to the console') do
+            options[:password_echo] = true
+          end
+        end
+      end
+    end
+  end
+end; end

data/lib/cloudflock/app/server-profile.rb

@@ -0,0 +1,130 @@
+require 'cloudflock/app/common/servers'
+require 'cloudflock/task/server-profile'
+require 'cloudflock/app'
+
+module CloudFlock; module App
+  # Public: The ServerProfile class provides the interface to produce profiles
+  # describing hosts running Unix-like operating systems as a CLI application.
+  class ServerProfile
+    include CloudFlock::App::Common
+    include CloudFlock::Remote
+
+    # Public: Connect to and profile a remote host, then display the gathered
+    # information.
+    def initialize
+      options = parse_options
+
+      source_host = define_source(options)
+      source_ssh = UI.spinner("Logging in to #{source_host[:hostname]}") do
+        SSH.new(source_host)
+      end
+
+      profile = UI.spinner("Checking source host") do
+        CloudFlock::Task::ServerProfile.new(source_ssh)
+      end
+
+      puts generate_report(profile)
+      puts profile.process_list if options[:verbose]
+    end
+
+    private
+
+    # Internal: Generate a "title" String (bold, 15 characters wide).
+    #
+    # tag - String to be turned into a title.
+    #
+    # Returns a String.
+    def title(tag)
+      UI.bold { "%15s" % tag }
+    end
+
+    # Internal: Generate a report containing informational aspects of a host's
+    # profile as well as any warnings profiling the host in question generated.
+    #
+    # profile - Profile object.
+    #
+    # Returns a String.
+    def generate_report(profile)
+      profile_hash = profile.to_hash
+      host_info(profile_hash[:info]) + host_warnings(profile_hash[:warnings])
+    end
+
+    # Internal: Generate a string containing informational aspects of a host's
+    # profile.
+    #
+    # profile - Profile object.
+    #
+    # Returns a String.
+    def host_info(profile)
+      profile.map do |section|
+        "#{UI.blue { UI.bold { section.title } } }\n" +
+        section.entries.reject do |entry|
+          entry.values.to_s.empty?
+        end.
+        map { |entry| title(entry.name) + " #{entry.values}" }.join("\n")
+      end.join("\n\n")
+    end
+
+    # Internal: Generate a string containing each warning produced by profiling
+    # a host.
+    #
+    # warnings - Array containing Strings.
+    #
+    # Returns a String.
+    def host_warnings(warnings)
+        warnings = warnings.map do |entry|
+           "* #{entry}"
+        end.join("\n")
+
+      unless warnings.empty?
+        warnings = UI.red { UI.bold { "\n\nWarnings:\n#{warnings}" } }
+      end
+      warnings
+    end
+
+    # Internal: Set up an OptionParser object to recognize options specific to
+    # profiling a remote host.
+    #
+    # Returns nothing.
+    def parse_options
+      options = {}
+
+      CloudFlock::App.parse_options(options) do |opts|
+        opts.separator 'Generate a report for a host'
+        opts.separator ''
+
+        opts.on('-h', '--host HOST', 'Target host to profile') do |host|
+          options[:hostname] = host
+        end
+
+        opts.on('-p', '--port PORT', 'Port SSH is listening on') do |port|
+          options[:port] = port
+        end
+
+        opts.on('-u', '--user USER', 'Username to log in') do |user|
+          options[:username] = user
+        end
+
+        opts.on('-a', '--password [PASSWORD]', 'Password to log in') do |pass|
+          options[:password] = pass
+        end
+
+        opts.on('-s', '--sudo', 'Use sudo to gain root') do
+          options[:sudo] = true
+        end
+
+        opts.on('-n', '--no-sudo', 'Use su to gain root') do
+          options[:sudo] = false
+        end
+
+        opts.on('-r', '--root-pass PASS', 'Password for root user') do |root|
+          options[:root_password] = root
+        end
+
+        opts.on('-i', '--identity IDENTITY', 'SSH identity to use') do |key|
+          options[:ssh_key] = key
+        end
+      end
+    end
+  end
+end; end