lxd-common 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 357a631e4d4f57d4da40fa8fd0be107475fe5884
-  data.tar.gz: dbe4e06ab24215c9a33a05945d43b87b410e4d1b
+  metadata.gz: 3ca7ef26b1a5f3811c395b2c90ab29bf5f0e8127
+  data.tar.gz: 56e6fd7f480fd6278bb011ba1052bebcea703172
 SHA512:
-  metadata.gz: 4ce7ae30122190b280fcbc337d6c0d822f55529e1a854132add4eb098a6d2808b87ed0534ad704710e928c31a9849d8b45323ac2037d1c11beb2f1c4190a937c
-  data.tar.gz: 5c18e9da89372e41c6484a88f9118ba0d9f555a636361ad7c84c1ad720610135c25a592f51cdc0254e2e0e7f751a34e0d635497c4b207e58d1e1454e854687ac
+  metadata.gz: c2719010fc06f675be0ae6c8720ecb5144119f962a4be36392a4b8e9c404532f784c81a3a3ef237a761d5cb70ca2dea2b8cf3afd3c7f6a62dfce11724d56b311
+  data.tar.gz: ffccf9fa39e661c337be6e04145da71dda189612db57fb87e21aba68e185e97f476af06e0f5b84517b5a4547b3e89436f61d0afea5277d5e1b20e66170a938eb

data/lib/nexussw/lxd/driver/mixins/cli.rb

@@ -1,3 +1,5 @@
+require 'nexussw/lxd/driver/mixins/helpers/wait'
+require 'nexussw/lxd/transport/cli'
 require 'tempfile'
 require 'yaml'
 require 'json'
@@ -14,6 +16,10 @@ module NexusSW
 
           attr_reader :inner_transport, :driver_options
 
+          def transport_for(container_name)
+            Transport::CLI.new inner_transport, container_name, info: YAML.load(inner_transport.execute('lxc info').error!.stdout)
+          end
+
           def create_container(container_name, container_options = {})
             if container_exists? container_name
               start_container container_name # Start for Parity with the below logic (`lxc launch` auto starts)
@@ -109,7 +115,7 @@ module NexusSW
             false
           end
 
-          include WaitMixin
+          include Helpers::WaitMixin
 
           protected
 

data/lib/nexussw/lxd/driver/mixins/helpers/wait.rb

@@ -0,0 +1,39 @@
+module NexusSW
+  module LXD
+    class Driver
+      module Mixins
+        module Helpers
+          module WaitMixin
+            def check_for_ip(driver, container_name)
+              cc = driver.container(container_name)
+              state = driver.container_state(container_name)
+              cc[:expanded_devices].each do |nic, data|
+                next unless data[:type] == 'nic'
+                state[:network][nic][:addresses].each do |address|
+                  return address[:address] if address[:family] == 'inet' && address[:address] && !address[:address].empty?
+                end
+              end
+              nil
+            end
+
+            def wait_for(container_name, what, timeout = 60)
+              Timeout.timeout timeout do
+                loop do
+                  retval = nil
+                  case what
+                  when :ip
+                    retval = check_for_ip(self, container_name)
+                  else
+                    raise 'unrecognized option'
+                  end
+                  return retval if retval
+                  sleep 0.5
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/driver/mixins/rest.rb

@@ -1,3 +1,5 @@
+require 'nexussw/lxd/driver/mixins/helpers/wait'
+require 'nexussw/lxd/transport/rest'
 require 'hyperkit'
 
 module NexusSW
@@ -17,19 +19,25 @@ module NexusSW
               auto_sync: true
             )
             @hk = inner_driver || Hyperkit::Client.new(hkoptions)
-            # HACK: can't otherwise get at the request timeout because sawyer is in the way
-            # Beware of unused function in hyperkit: reset_agent  If that gets used it'll undo this timeout
-            # unneeded while default valued: @hk.agent.instance_variable_get(:@conn).options[:timeout] = REQUEST_TIMEOUT
           end
 
           attr_reader :hk, :rest_endpoint, :driver_options
 
+          include Helpers::WaitMixin
+
+          def server_info
+            @server_info ||= hk.get('/1.0')[:metadata]
+          end
+
+          def transport_for(container_name)
+            Transport::Rest.new container_name, info: server_info, connection: hk, driver_options: driver_options, rest_endpoint: rest_endpoint
+          end
+
           def create_container(container_name, container_options = {})
             if container_exists?(container_name)
               start_container container_name # Start the container for Parity with the CLI
               return container_name
             end
-            # we'll break this apart and time it out for those with slow net (and this was my 3 minute stress test case with good net)
             # parity note: CLI will run indefinitely rather than timeout hence the 0 timeout
             retry_forever do
               @hk.create_container(container_name, container_options.merge(sync: false))
@@ -48,25 +56,29 @@ module NexusSW
 
           def stop_container(container_id, options = {})
             return if container_status(container_id) == 'stopped'
-            return @hk.stop_container(container_id, force: true) if options[:force]
-            last_id = nil
-            use_last = false
-            LXD.with_timeout_and_retries({ timeout: 0 }.merge(options)) do # timeout: 0 to enable retry functionality
-              return if container_status(container_id) == 'stopped'
-              begin
-                unless use_last
-                  # Keep resubmitting until the server complains (Stops will be ignored/hang if init is not yet listening for SIGPWR i.e. recently started)
-                  begin
-                    last_id = @hk.stop_container(container_id, sync: false)[:id] # TODO: this 'could' hang for 2 minutes?  and then we'd never get a last_id if that's where the hang happens, and then it 'could' error on retry
-                  rescue Hyperkit::BadRequest # Happens if a stop command has previously been accepted as well as other reasons.  handle that on next line
-                    raise unless last_id # if we have a last_id then a prior stop command has successfully initiated so we'll just wait on that one
-                    use_last = true
+            if options[:force]
+              @hk.stop_container(container_id, force: true)
+            else
+              last_id = nil
+              use_last = false
+              LXD.with_timeout_and_retries({ timeout: 0 }.merge(options)) do # timeout: 0 to enable retry functionality
+                return if container_status(container_id) == 'stopped'
+                begin
+                  unless use_last
+                    # Keep resubmitting until the server complains (Stops will be ignored/hang if init is not yet listening for SIGPWR i.e. recently started)
+                    begin
+                      last_id = @hk.stop_container(container_id, sync: false)[:id]
+                    rescue Hyperkit::BadRequest # Happens if a stop command has previously been accepted as well as other reasons.  handle that on next line
+                      # if we have a last_id then a prior stop command has successfully initiated so we'll just wait on that one
+                      raise unless last_id # rubocop:disable Metrics/BlockNesting
+                      use_last = true
+                    end
                   end
+                  @hk.wait_for_operation last_id # , options[:retry_interval]
+                rescue Faraday::TimeoutError => e
+                  return if container_status(container_id) == 'stopped'
+                  raise Timeout::Retry.new e # if options[:retry_interval] # rubocop:disable Style/RaiseArgs
                 end
-                @hk.wait_for_operation last_id # , options[:retry_interval]
-              rescue Faraday::TimeoutError => e
-                return if container_status(container_id) == 'stopped'
-                raise Timeout::Retry.new e # if options[:retry_interval] # rubocop:disable Style/RaiseArgs
               end
             end
             wait_for_status container_id, 'stopped'
@@ -74,8 +86,14 @@ module NexusSW
 
           def delete_container(container_id)
             return unless container_exists? container_id
-            stop_container container_id, force: true
-            @hk.delete_container(container_id)
+            stop_container container_id,  force: true
+
+            # overcome a race condition where the host is busy doing 'something' async causing the delete command to pop an error
+            # https://github.com/lxc/lxd/issues/4063
+            # sleep 1
+
+            id = @hk.delete_container(container_id, sync: false)[:id]
+            @hk.wait_for_operation id
           end
 
           def container_status(container_id)
@@ -98,8 +116,6 @@ module NexusSW
             false
           end
 
-          include WaitMixin
-
           protected
 
           def wait_for_status(container_id, newstatus)

data/lib/nexussw/lxd/driver.rb

@@ -22,67 +22,39 @@ module NexusSW
       }.freeze
 
       def create_container(_container_name, _container_options)
-        raise 'NexusSW::LXD::Driver.create_container not implemented'
+        raise "#{self.class}#create_container not implemented"
       end
 
       def start_container(_container_id)
-        raise 'NexusSW::LXD::Driver.start_container not implemented'
+        raise "#{self.class}#start_container not implemented"
       end
 
       def stop_container(_container_id, _options = {})
-        raise 'NexusSW::LXD::Driver.stop_container not implemented'
+        raise "#{self.class}#stop_container not implemented"
       end
 
       def delete_container(_container_id)
-        raise 'NexusSW::LXD::Driver.delete_container not implemented'
+        raise "#{self.class}#delete_container not implemented"
       end
 
       def container_status(_container_id)
-        raise 'NexusSW::LXD::Driver.container_status not implemented'
-      end
-
-      def ensure_profiles(_profiles)
-        raise 'NexusSW::LXD::Driver.ensure_profiles not implemented'
+        raise "#{self.class}#container_status not implemented"
       end
 
       def container(_container_id)
-        raise 'NexusSW::LXD::Driver.container not implemented'
+        raise "#{self.class}#container not implemented"
       end
 
       def container_state(_container_id)
-        raise 'NexusSW::LXD::Driver.container_state not implemented'
+        raise "#{self.class}#container_state not implemented"
       end
 
       def wait_for(_what)
-        raise 'NexusSW::LXD::Driver.wait_for not implemented'
+        raise "#{self.class}#wait_for not implemented"
       end
 
-      module WaitMixin
-        def check_for_ip(driver, container_name)
-          cc = driver.container(container_name)
-          state = driver.container_state(container_name)
-          cc[:expanded_devices].each do |nic, data|
-            next unless data[:type] == 'nic'
-            state[:network][nic][:addresses].each do |address|
-              return address[:address] if address[:family] == 'inet' && address[:address] && !address[:address].empty?
-            end
-          end
-          nil
-        end
-
-        def wait_for(container_name, what, timeout = 60)
-          Timeout.timeout timeout do
-            loop do
-              retval = nil
-              case what
-              when :ip
-                retval = check_for_ip(self, container_name)
-              end
-              return retval if retval
-              sleep 0.5
-            end
-          end
-        end
+      def transport_for(_container_name)
+        raise "#{self.class}#transport_for not implemented"
       end
     end
   end

data/lib/nexussw/lxd/transport/mixins/cli.rb

@@ -1,4 +1,5 @@
 require 'nexussw/lxd/transport/mixins/local'
+require 'nexussw/lxd/transport/mixins/helpers/upload_folder'
 require 'tempfile'
 require 'pp'
 
@@ -13,8 +14,11 @@ module NexusSW
             @inner_transport = remote_transport
             @punt = !inner_transport.is_a?(::NexusSW::LXD::Transport::Mixins::Local)
           end
+
           attr_reader :inner_transport, :punt, :container_name, :config
 
+          include Helpers::UploadFolder
+
           def execute(command, options = {})
             mycommand = command.is_a?(Array) ? command.join(' ') : command
             subcommand = options[:subcommand] || "exec #{container_name} --"
@@ -62,6 +66,11 @@ module NexusSW
             inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
           end
 
+          def upload_folder(local_path, path)
+            return super unless config[:info] && config[:info][:api_extensions] && config[:info][:api_extensions].include?('directory_manipulation')
+            execute("-r #{localname} #{container_name}#{path}", subcommand: 'file push', capture: false).error!
+          end
+
           def add_remote(host_name)
             execute("add #{host_name} --accept-certificate", subcommand: 'remote').error! unless remote? host_name
           end

data/lib/nexussw/lxd/transport/mixins/helpers/execute.rb

@@ -0,0 +1,53 @@
+module NexusSW
+  module LXD
+    class Transport
+      module Mixins
+        module Helpers
+          module ExecuteMixin
+            class ExecuteResult
+              def initialize(command, options, exitstatus)
+                @command = command
+                @options = options || {}
+                @exitstatus = exitstatus
+              end
+
+              attr_reader :exitstatus, :options, :command
+
+              def stdout
+                options[:capture_options][:stdout] if options.key? :capture_options
+              end
+
+              def stderr
+                options[:capture_options][:stderr] if options.key? :capture_options
+              end
+
+              def error!
+                return self if exitstatus == 0
+                msg = "Error: '#{command}' failed with exit code #{exitstatus}.\n"
+                msg += "STDOUT: #{stdout}" if stdout && !stdout.empty?
+                msg += "STDERR: #{stderr}" if stderr && !stderr.empty?
+                raise msg
+              end
+            end
+
+            def execute(command, options = {}, &block)
+              options ||= {}
+              return execute_chunked(command, options) if options[:capture] == false && !block_given?
+
+              capture_options = { stdout: '', stderr: '' }
+              capture_options[:capture] = block if block_given?
+              capture_options[:capture] ||= options[:capture] if options[:capture].respond_to? :call
+              # capture_options[:capture] ||= options[:stream] if options[:stream].respond_to? :call
+              capture_options[:capture] ||= proc do |stdout_chunk, stderr_chunk|
+                capture_options[:stdout] += stdout_chunk if stdout_chunk
+                capture_options[:stderr] += stderr_chunk if stderr_chunk
+              end
+
+              execute_chunked(command, options.merge(capture_options: capture_options), &capture_options[:capture])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/transport/mixins/helpers/upload_folder.rb

@@ -0,0 +1,79 @@
+module NexusSW
+  module LXD
+    class Transport
+      module Mixins
+        module Helpers
+          module UploadFolder
+            def upload_folder(local_path, path)
+              upload_using_tarball(local_path, path) || upload_files_individually(local_path, path)
+            end
+
+            def upload_files_individually(local_path, path)
+              Dir.entries(local_path).map { |f| (f == '.' || f == '..') ? nil : File.join(local_path, f) }.compact.each do |f|
+                dest = File.join(path, File.basename(local_path))
+                upload_files_individually f, dest if File.directory? f
+                upload_file f, File.join(dest, File.basename(f)) if File.file? f
+              end
+            end
+
+            def upload_using_tarball(local_path, path)
+              return false unless can_archive?
+              # TODO: should I return false upon error?  i.e. retry with individual file uploads if this fails?
+              #   lets see how this does in the wild before deciding
+              flag, ext = compression
+              begin
+                tfile = Tempfile.new(container_name)
+                tfile.close
+                `tar -c#{flag}f #{tfile.path} -C #{File.dirname local_path} ./#{File.basename local_path}`
+                # on that above note we'll do this at least
+                # raise "Unable to create archive #{tfile.path}" if File.zero? tfile.path
+                if File.zero? tfile.path
+                  @can_archive = false
+                  return false
+                end
+                fname = '/tmp/' + File.basename(tfile.path) + ".tar#{ext}"
+                upload_file tfile.path, fname
+                # TODO: serious: make sure the tar extract does an overwrite of existing files
+                #   multiple converge support as well as CI cycle/dev updated files get updated instead of .1 suffixed (?)
+                #   I think I need a flag (it's been a while)
+                execute("bash -c 'mkdir -p #{path} && cd #{path} && tar -xf #{fname} && rm -rf #{fname}'", capture: false).error!
+              ensure
+                tfile.unlink
+              end
+            end
+
+            private
+
+            def can_archive?
+              return false if @can_archive == false
+              @can_archive ||= begin
+                                  # I don't want to code tarball logic into the mock transport
+                                  return false if respond_to?(:hk) && hk.respond_to?(:mock)
+                                  return false if respond_to?(:inner_transport) && inner_transport.respond_to?(:mock)
+                                  return false if respond_to?(:inner_transport) && inner_transport.respond_to?(:inner_transport) && inner_transport.inner_transport.respond_to?(:mock)
+                                  return false if respond_to?(:inner_transport) && inner_transport.respond_to?(:hk) && inner_transport.hk.respond_to?(:mock)
+                                  `tar --version`
+                                  true
+                                rescue
+                                  false
+                                end
+            end
+
+            # gzip(-z) or bzip2(-j) (these are the only 2 on trusty atm)
+            def compression
+              @compression ||= begin
+                                  which = execute('bash -c "which gzip || which bzip2 || true"').stdout.strip
+                                  which = File.basename(which) if which
+                                  case which
+                                  when 'gzip' then ['z', '.gz']
+                                  when 'bzip2' then ['j', '.bzip2']
+                                  else ['', '']
+                                  end
+                                end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/transport/mixins/local.rb

@@ -1,3 +1,4 @@
+require 'nexussw/lxd/transport/mixins/helpers/execute'
 require 'open3'
 require 'nio/websocket'
 
@@ -12,7 +13,7 @@ module NexusSW
 
           attr_reader :config
 
-          include ExecuteMixin
+          include Helpers::ExecuteMixin
 
           def execute_chunked(command, options)
             NIO::WebSocket::Reactor.start
@@ -36,7 +37,7 @@ module NexusSW
                 end
                 th.join
                 loop do
-                  return LXDExecuteResult.new(command, options, th.value.exitstatus) if th.value.exited? && mon_out && mon_err && mon_out.closed? && mon_err.closed?
+                  return Helpers::ExecuteMixin::ExecuteResult.new(command, options, th.value.exitstatus) if th.value.exited? && mon_out && mon_err && mon_out.closed? && mon_err.closed?
                   Thread.pass
                 end
               end

data/lib/nexussw/lxd/transport/mixins/rest.rb

@@ -1,22 +1,26 @@
+require 'nexussw/lxd/transport/mixins/helpers/execute'
+require 'nexussw/lxd/transport/mixins/helpers/upload_folder'
 require 'nio/websocket'
+require 'tempfile'
 
 module NexusSW
   module LXD
     class Transport
       module Mixins
         module Rest
-          def initialize(driver, container_name, config = {})
+          def initialize(container_name, config = {})
             @container_name = container_name
             @config = config
-            raise "The rest transport requires the Rest Driver.  You supplied #{driver}" unless driver.respond_to?(:hk) && driver.respond_to?(:rest_endpoint) # driver.is_a? NexusSW::LXD::Driver::Rest
-            @rest_endpoint = driver.rest_endpoint
-            @driver_options = driver.driver_options
-            @hk = driver.hk
+            @rest_endpoint = config[:rest_endpoint]
+            @driver_options = config[:driver_options]
+            @hk = config[:connection]
+            raise 'The rest transport requires the following keys: { :connection, :driver_options, :rest_endpoint }' unless @rest_endpoint && @hk && @driver_options
           end
 
           attr_reader :hk, :rest_endpoint, :container_name, :config
 
-          include ExecuteMixin
+          include Helpers::ExecuteMixin
+          include Helpers::UploadFolder
 
           def execute_chunked(command, options = {}, &block)
             opid = nil
@@ -32,7 +36,7 @@ module NexusSW
               begin
                 retval = hk.wait_for_operation opid
                 backchannel.exit if backchannel.respond_to? :exit
-                return LXDExecuteResult.new command, options, retval[:metadata][:return].to_i
+                return Helpers::ExecuteMixin::ExecuteResult.new command, options, retval[:metadata][:return].to_i
               rescue Faraday::TimeoutError => e
                 raise Timeout::Retry.new e # rubocop:disable Style/RaiseArgs
               end
@@ -54,7 +58,8 @@ module NexusSW
           end
 
           def upload_file(local_path, path)
-            hk.push_file local_path, container_name, path
+            # return hk.push_file(local_path, container_name, path)
+            write_file(path, IO.binread(local_path))
           end
 
           protected

data/lib/nexussw/lxd/transport.rb

@@ -3,71 +3,29 @@ require 'nexussw/lxd'
 module NexusSW
   module LXD
     class Transport
-      class LXDExecuteResult
-        def initialize(command, options, exitstatus)
-          @command = command
-          @options = options || {}
-          @exitstatus = exitstatus
-        end
-
-        attr_reader :exitstatus, :options, :command
-
-        def stdout
-          options[:capture_options][:stdout] if options.key? :capture_options
-        end
-
-        def stderr
-          options[:capture_options][:stderr] if options.key? :capture_options
-        end
-
-        def error!
-          return self if exitstatus == 0
-          msg = "Error: '#{command}' failed with exit code #{exitstatus}.\n"
-          msg += "STDOUT: #{stdout}" if stdout && !stdout.empty?
-          msg += "STDERR: #{stderr}" if stderr && !stderr.empty?
-          raise msg
-        end
-      end
-
-      module ExecuteMixin
-        def execute(command, options = {}, &block)
-          options ||= {}
-          return execute_chunked(command, options) if options[:capture] == false && !block_given?
-
-          capture_options = { stdout: '', stderr: '' }
-          capture_options[:capture] = block if block_given?
-          capture_options[:capture] ||= options[:capture] if options[:capture].respond_to? :call
-          # capture_options[:capture] ||= options[:stream] if options[:stream].respond_to? :call
-          capture_options[:capture] ||= proc do |stdout_chunk, stderr_chunk|
-            capture_options[:stdout] += stdout_chunk if stdout_chunk
-            capture_options[:stderr] += stderr_chunk if stderr_chunk
-          end
-
-          execute_chunked(command, options.merge(capture_options: capture_options), &capture_options[:capture])
-        end
-      end
-
       def execute(_command, _options = {})
-        raise 'NexusSW::LXD::Transport.execute not implemented'
+        raise "#{self.class}#execute not implemented"
       end
 
       def read_file(_path)
-        raise 'NexusSW::LXD::Transport.read_file not implemented'
+        raise "#{self.class}#read_file not implemented"
       end
 
       def write_file(_path, _content)
-        raise 'NexusSW::LXD::Transport.write_file not implemented'
+        raise "#{self.class}#write_file not implemented"
       end
 
       def download_file(_path, _local_path)
-        raise 'NexusSW::LXD::Transport.download_file not implemented'
+        raise "#{self.class}#download_file not implemented"
       end
 
       def upload_file(_local_path, _path)
-        raise 'NexusSW::LXD::Transport.upload_file not implemented'
+        raise "#{self.class}#upload_file not implemented"
+      end
+
+      def upload_folder(_local_path, _path)
+        raise "#{self.class}#upload_folder not implemented"
       end
-      # protected
-      # def execute_chunked(_command, _options = {})
     end
   end
 end

data/lib/nexussw/lxd/version.rb

@@ -1,5 +1,5 @@
 module NexusSW
   module LXD
-    VERSION = '0.4.1'.freeze
+    VERSION = '0.5.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lxd-common
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Sean Zachariasen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-08 00:00:00.000000000 Z
+date: 2017-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hyperkit
@@ -115,12 +115,15 @@ files:
 - lib/nexussw/lxd/driver.rb
 - lib/nexussw/lxd/driver/cli.rb
 - lib/nexussw/lxd/driver/mixins/cli.rb
+- lib/nexussw/lxd/driver/mixins/helpers/wait.rb
 - lib/nexussw/lxd/driver/mixins/rest.rb
 - lib/nexussw/lxd/driver/rest.rb
 - lib/nexussw/lxd/transport.rb
 - lib/nexussw/lxd/transport/cli.rb
 - lib/nexussw/lxd/transport/local.rb
 - lib/nexussw/lxd/transport/mixins/cli.rb
+- lib/nexussw/lxd/transport/mixins/helpers/execute.rb
+- lib/nexussw/lxd/transport/mixins/helpers/upload_folder.rb
 - lib/nexussw/lxd/transport/mixins/local.rb
 - lib/nexussw/lxd/transport/mixins/rest.rb
 - lib/nexussw/lxd/transport/rest.rb