cloudflock 0.7.0 → 0.7.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NjNmYTg2NmMyOTExM2M1YjlkM2ExMjkzMDE0Yzc1ZjlhYzNmNGIyOQ==
-  data.tar.gz: !binary |-
-    N2Q2M2Y4ZjMyNmE2MDYzNTA3MjRmYTYyNTQ0OWRlNDIyYWY5ODdhOQ==
+SHA1:
+  metadata.gz: 421fa1e64759c72ef901ee7431000811e635e4d1
+  data.tar.gz: 570f428da5e21ae71020b3eaf6d9e93a5c5a9e9c
 SHA512:
-  metadata.gz: !binary |-
-    MTZlYTlmOWMwMTMwM2IxMTRjOWFjY2Y1OGNlMWQ4YjE2OGZkMDUyNjMxNDI1
-    NDU1OTczYTBhNzI3OTYzNjhlY2QwNDI5NTQ0NTE4MzExM2NiMjkwNWZmZWUy
-    Y2U1ZjJkZjgyZmJkMTViMzMzMTBlZjljMGUxNzRjN2IxMjM2YTg=
-  data.tar.gz: !binary |-
-    MWY2ZGYyMDUwYTFkYzZmZDhkMDZiMmQ5NDU1ZDA3Y2U0MzhiYWFjNDQ0NTc4
-    NDYyZjM4ZjNhMWVlZTU1OTUxNTk2MzhlODgyMDRhY2UxYjRlYjhjZjgwNzMw
-    NTlhYmFlMjQ5YzQ0NzY3ZjA3MmJmNzJkNWFjMDc1M2VjZTEyZTE=
+  metadata.gz: 444ea7d4d45b7d614f41005f69e606d32d8a92a0e44d0f75d7291e14770acf75af8799cd45d88b216f6c13f6dcb3f67495844c85393ddf3c799c83aab032a76a
+  data.tar.gz: 30714d89645a257a8bc7481d05d5700a8d54f433d484335b7caabc24c3464a35d735e5d15d5f77fbd2e465c0bed6d00f60c2a5a4bdbe309911f8c9b0c0f2ed78

data/lib/cloudflock/app/common/rackspace.rb

@@ -13,23 +13,29 @@ module CloudFlock; module App
     # Returns a Hash containing information necessary to establish an API
     # session.
     def define_rackspace_api
-      api = { provider: 'rackspace' }
-
-      api[:rackspace_username] = UI.prompt('Rackspace Cloud Username')
-      api[:rackspace_api_key]  = UI.prompt('Rackspace Cloud API key')
-
-      region = UI.prompt('Account Region (us, uk)',
-                         valid_answers: [/^u[sk]$/i])
-      region.gsub!(/$/, '_AUTH_ENDPOINT')
-      api[:rackspace_region] = Fog::Rackspace.const_get(region.upcase)
+      {
+        provider:           'rackspace',
+        rackspace_username: UI.prompt('Rackspace Cloud Username'),
+        rackspace_api_key:  UI.prompt('Rackspace Cloud API key'),
+        rackspace_region:   define_rackspace_region
+      }
+    end
 
-      api
+    # Public: Determine which Rackspace public endpoint should be used.
+    #
+    # Returns a String.
+    def define_rackspace_region
+      countries = [/^u[sk]$/i]
+      region = UI.prompt('Account Region (us, uk)', valid_answers: countries)
+      Fog::Rackspace.const_get(region.upcase + '_AUTH_ENDPOINT')
     end
 
     # Public: Wrap define_rackspace_service_region, specifying
     # 'cloudServersOpenStack' as the service type.
     #
     # api     - Authenticated Fog API instance.
+    #
+    # Returns a Hash.
     def define_rackspace_cloudservers_region(api)
       api.merge(define_rackspace_service_region(api, 'cloudServersOpenStack'))
     end
@@ -38,6 +44,8 @@ module CloudFlock; module App
     # the service type.
     #
     # api     - Authenticated Fog API instance.
+    #
+    # Returns a Hash.
     def define_rackspace_files_region(api)
       api.merge(define_rackspace_service_region(api, 'cloudFiles'))
     end

data/lib/cloudflock/app/common/servers.rb

@@ -76,8 +76,8 @@ module CloudFlock; module App
 
       key_path = File.join(Dir.home, '.ssh', 'id_rsa')
       key_path = '' unless File.exists?(key_path)
-      check_option(host, :ssh_key, "#{name} SSH Key",
-                   default_answer: key_path, allow_empty: true)
+      check_option_fs(host, :ssh_key, "#{name} SSH Key",
+                      default_answer: key_path, allow_empty: true)
 
       # Using sudo is only applicable if the user isn't root
       host[:sudo] = false if host[:username] == 'root'

data/lib/cloudflock/app/files-migrate.rb

@@ -52,30 +52,43 @@ module CloudFlock; module App
     #
     # Returns a CloudFlock::Remote::Files object.
     def define_api(desc, create = false)
-      store   = {}
       query   = "#{desc} provider (rackspace, aws, local)"
       answers = [/^(?:rackspace|aws|local)$/i]
 
       provider = UI.prompt(query, valid_answers: answers)
 
-      case provider
+      api = case provider
       when /rackspace/i
-        store[:provider] = 'Rackspace'
-        store[:rackspace_username] = UI.prompt('Rackspace username')
-        store[:rackspace_api_key] = UI.prompt('Rackspace API key')
+        define_rackspace_api
       when /aws/i
-        store[:provider] = 'AWS'
-        store[:aws_access_key_id] = UI.prompt('AWS Access Key ID')
-        store[:aws_secret_access_key] = UI.prompt('AWS secret access key')
+        {
+          provider: 'AWS',
+          aws_access_key_id: UI.prompt('AWS Access Key ID'),
+          aws_secret_access_key: UI.prompt('AWS secret access key')
+        }
       when /local/i
-        store[:provider] = 'local'
-        store[:local_root] = UI.prompt("#{desc} location")
-        return CloudFlock::Remote::Files.new(store)
+        {
+          provider:   'local',
+          local_root: UI.prompt("#{desc} location")
+        }
       end
 
-      api = CloudFlock::Remote::Files.new(store)
+      store = api.merge(define_rackspace_files_region(api))
+      setup_object_store(CloudFlock::Remote::Files.new(store), desc, create)
+    end
 
-      options = api.directories.map do |dir|
+    # Internal: Connect to an object store and determine the directory on which
+    # to act.
+    #
+    # store  - Fog::Remote::Files object.
+    # desc   - Description of the data store for display purposes.
+    # create - Whether to create non-existing locations.
+    #
+    # Returns a CloudFlock::Remote::Files object.
+    def setup_object_store(store, desc, create)
+      return store if store.local?
+
+      options = store.directories.map do |dir|
         { name: dir.key, files: dir.count.to_s }
       end
       valid = options.reduce([]) { |c,e| c << e[:name] }
@@ -83,15 +96,15 @@ module CloudFlock; module App
       puts UI.build_grid(options, name: "Directory name", files: "File count")
       if create
         selected = UI.prompt("#{desc} directory")
-        unless api.directories.select { |dir| dir.key == selected }.any?
-          api.directories.create(key: selected)
+        unless store.directories.select { |dir| dir.key == selected }.any?
+          store.directories.create(key: selected)
         end
       else
         selected = UI.prompt("#{desc} directory", valid_answers: valid)
       end
-      api.directory = selected
+      store.directory = selected
 
-      api
+      store
     end
 
     # Internal: Set up queue and Mutexes, create threads to manage the transfer
@@ -111,34 +124,35 @@ module CloudFlock; module App
     #
     # Returns nothing.
     def files_migrate(source_store, dest_store, options = {})
-      file_queue       = []
-      mutexes          = { queue: Mutex.new, finished: Mutex.new }
-      up_threads       = options[:upload_threads] || UPLOAD_THREADS
-      down_threads     = options[:download_threads] || DOWNLOAD_THREADS
+      mutexes      = { queue: Queue.new, ongoing: Mutex.new }
+      up_threads   = options[:upload_threads]   || UPLOAD_THREADS
+      down_threads = options[:download_threads] || DOWNLOAD_THREADS
 
-      source = Thread.new do
-        manage_source(source_store, file_queue, mutexes, up_threads)
-      end
+      mutexes[:ongoing].lock
 
       destination = Thread.new do
-        manage_destination(dest_store, file_queue, mutexes, down_threads)
+        manage_destination(dest_store, mutexes, down_threads)
+      end
+      source = Thread.new do
+        manage_source(source_store, mutexes, up_threads)
       end
 
-      [source, destination].each(&:join)
+      [source].each(&:join)
+      mutexes[:ongoing].unlock
+
+      [destination].each(&:join)
     end
 
     # Internal: Create and observe threads which download files from a
     # non-local file store.  If the files exist locally, simply generate a list
     # of the files in queue.
     #
-    # source_store - CloudFlock::Remote::Files object set up to pull files from
+    # store        - CloudFlock::Remote::Files object set up to pull files from
     #                a source directory.
-    # file_queue   - Array in which details regarding files to be transferred
-    #                will be stored.
     # mutexes      - Hash containing two Mutexes:
-    #                :queue    - Coordinates access to file_queue.
-    #                :finished - Locked immediately, only unlocked once all
-    #                            files are queued for transfer.
+    #                :queue   - Queue to coordinate file access.
+    #                :ongoing - Indicates that the transfer is ongoing when
+    #                           locked.
     # thread_count - Hash optionally containing overrides for the number of
     #                upload and download threads to use for transfer
     #                concurrency. (default: {})
@@ -148,38 +162,30 @@ module CloudFlock; module App
     #                                    Overrides DOWNLOAD_THREADS constant.
     #
     # Returns nothing.
-    def manage_source(source_store, file_queue, mutexes, thread_count)
-      mutexes[:finished].lock
-
-      if source_store.local?
-        source_store.each_file do |file|
-          mutexes[:queue].synchronize do
-            file_queue << { path: "#{source_store.prefix}/#{file.key}",
-                            name: file.key, temp: false }
-          end
+    def manage_source(store, mutexes, thread_count)
+      if store.local?
+        store.each_file do |file|
+          node = File.new("#{store.prefix}/#{file.key}")
+          mutexes[:queue] << { file: node, name: file.key, temp: false }
         end
       else
         source_threads  = []
         file_list_mutex = Mutex.new
-        file_list       = source_store.file_list
+        file_list       = store.file_list
 
         thread_count.times do
           source_threads << Thread.new do
             while file = file_list_mutex.synchronize { file_list.pop } do
               temp = Tempfile.new(file.gsub(/\//, ''))
-              temp.write(source_store.get_file(file))
+              temp.write(store.get_file(file))
               temp.close
-              mutexes[:queue].synchronize do
-                file_queue << { path: temp.path, name: file, temp: true }
-              end
+              mutexes[:queue] << { file: temp, name: file, temp: true }
             end
           end
         end
 
         source_threads.each(&:join)
       end
-
-      mutexes[:finished].unlock
     end
 
     # Internal: Create and observe threads which download files from a
@@ -188,12 +194,10 @@ module CloudFlock; module App
     #
     # dest_store   - CloudFlock::Remote::Files object set up to upload files to
     #                a destination directory.
-    # file_queue   - Array from which to retrieve details regarding files to be
-    #                transferred.
     # mutexes      - Hash containing two Mutexes:
-    #                :queue    - Coordinates access to file_queue.
-    #                :finished - Locked immediately, only unlocked once all
-    #                            files are queued for transfer.
+    #                :queue   - Queue to coordinate file access.
+    #                :ongoing - Indicates that the transfer is ongoing when
+    #                           locked.
     # thread_count - Hash optionally containing overrides for the number of
     #                upload and download threads to use for transfer
     #                concurrency. (default: {})
@@ -203,24 +207,41 @@ module CloudFlock; module App
     #                                    Overrides DOWNLOAD_THREADS constant.
     #
     # Returns nothing.
-    def manage_destination(dest_store, file_queue, mutexes, thread_count)
+    def manage_destination(dest_store, mutexes, thread_count)
       dest_threads = []
 
       thread_count.times do
         dest_threads << Thread.new do
-          while mutexes[:finished].locked?
-            while file = mutexes[:queue].synchronize { file_queue.pop }
-              content = File.read(file[:path])
-              dest_store.create(key: file[:name], body: content)
-              File.unlink(file[:path]) if file[:temp]
-            end
-          end
+          upload_thread(dest_store, mutexes)
         end
       end
 
       dest_threads.each(&:join)
     end
 
+    # Internal: Upload files from a Queue to a given object store.
+    #
+    # dest_store   - CloudFlock::Remote::Files object set up to upload files to
+    #                a destination directory.
+    # mutexes      - Hash containing two Mutexes:
+    #                :queue   - Queue to coordinate file access.
+    #                :ongoing - Indicates that the transfer is ongoing when
+    #                           locked.
+    #
+    # Returns nothing.
+    def upload_thread(dest_store, mutexes)
+      while mutexes[:ongoing].locked?
+        while file = mutexes[:queue].pop(true)
+          file[:file].open if file[:temp]
+          dest_store.create(key: file[:name], body: file[:file].read)
+          file[:file].close
+        end
+      end
+    rescue ThreadError
+      sleep 0.1
+      retry
+    end
+
     # Internal: Set up an OptionParser object to recognize options specific to
     # profiling a remote host.
     #

data/lib/cloudflock/app.rb

@@ -26,6 +26,24 @@ module CloudFlock
       options[name] = UI.prompt(prompt, prompt_options)
     end
 
+    # Public: Wrap check_option, allowing for filesystem autocompletion in user
+    # response if the option is not set.
+    #
+    # options        - Hash containing options to test against.
+    # name           - The key in the options Hash expected to contain the
+    #                  response desired.
+    # prompt         - Prompt to present to the user.
+    # prompt_options - Options to pass along to ConsoleGlitter::UI#prompt.
+    #                  (default: {})
+    #
+    # Returns the contents of the options[name] or else a String if
+    # options[name] is nil.
+    def check_option_fs(options, name, prompt, prompt_options = {})
+      return options[name] unless options[name].nil?
+
+      options[name] = UI.prompt_filesystem(prompt, prompt_options)
+    end
+
     # Public: Check if an option is set; return the value if so, otherwise
     # prompt the user for a response.
     #

data/lib/cloudflock/remote/files.rb

@@ -63,6 +63,9 @@ module CloudFlock; module Remote
     # Returns a String.
     def get_file(file)
       @files.files.get(file).body
+    rescue Excon::Errors::Timeout, Fog::Storage::Rackspace::ServiceError
+      # Triggered by server and request timeouts respectively.
+      retry
     end
 
     # Public: Create a file in the current directory.
@@ -74,7 +77,8 @@ module CloudFlock; module Remote
     # Returns nothing.
     def create(file_spec)
       @files.files.create(file_spec)
-    rescue Excon::Errors::Timeout
+    rescue Excon::Errors::Timeout, Fog::Storage::Rackspace::ServiceError
+      # Triggered by server and request timeouts respectively.
       retry
     end
 

data/lib/cloudflock.rb

@@ -4,5 +4,5 @@ require 'cloudflock/errstr'
 # Public: Encapsulate all functionality related to the CloudFlock API and any
 # apps built with such.
 module CloudFlock
-  VERSION = '0.7.0'
+  VERSION = '0.7.1'
 end

metadata

@@ -1,69 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: cloudflock
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Chris Wuest
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-10 00:00:00.000000000 Z
+date: 2014-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.21'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.21'
 - !ruby/object:Gem::Dependency
   name: cpe
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: console-glitter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
-    - - ! '>='
+        version: '0.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.4
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
-    - - ! '>='
+        version: '0.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.4
+        version: 0.2.1
 description: CloudFlock is a library and toolchain focused on migration
 email: chris@chriswuest.com
 executables:
@@ -111,12 +111,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []