lxd-common 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3ca7ef26b1a5f3811c395b2c90ab29bf5f0e8127
-  data.tar.gz: 56e6fd7f480fd6278bb011ba1052bebcea703172
+SHA256:
+  metadata.gz: 2862f300294934ad732a85a91460a901ddca30a1371b59f3c52d4823f845ba22
+  data.tar.gz: cf9f7a414093dce415a457d144ea222ae8bc5e106d18df00a9d0510c61876ce2
 SHA512:
-  metadata.gz: c2719010fc06f675be0ae6c8720ecb5144119f962a4be36392a4b8e9c404532f784c81a3a3ef237a761d5cb70ca2dea2b8cf3afd3c7f6a62dfce11724d56b311
-  data.tar.gz: ffccf9fa39e661c337be6e04145da71dda189612db57fb87e21aba68e185e97f476af06e0f5b84517b5a4547b3e89436f61d0afea5277d5e1b20e66170a938eb
+  metadata.gz: d21f7a64be74067ff98a0170f0acb5006c1827000c3cc5dbe87f944c74fd362fd1b20202b3936e36ea84544b53159122b565732065a63d764f2bee5838a185fc
+  data.tar.gz: 91f6e12a53b3255b53f04dd875777b79c548d01aa4c88fcae6e5a240a45071704e1ab62c84480e56bb251627e7054df675a9cebe697c7e5a93a2d0485f346267

data/.gitignore

@@ -1,6 +1,6 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
+*.lock
 /_yardoc/
 /coverage/
 /doc/

data/.travis.yml

@@ -9,7 +9,6 @@ matrix:
   allow_failures:
   - rvm: ruby-head
 before_install:
-- gem update --system
 - curl -L https://www.chef.io/chef/install.sh | sudo bash
 - sudo chef-apply spec/provision_recipe.rb
 - ip a && ip r
@@ -26,3 +25,11 @@ after_script:
 notifications:
   slack:
     secure: cMKi0hpXPZt8loD84alldIn3FCB0XBwHyOmh94MYnJ2GwtqGmrdDod5WLa36123z6XaDqpX0d/kXUZGWoVPTRK5QKWzlp/2bR6ZNGcWGFtB37UCjDweF9NRKgi56Nn2z3ws7qz/tG0MtH2991KjBh+Q5StaKHSTSh5BTHkLXFJ+YSW4ucY0PUctenF0sAz6vrLVNiYyrfWlGG6JBAd5LjinB6NdZHIDAl3yZ+IGrfxLXKAbLDgWcEVbXGOpQ0XSe7jiAzg7gi1ofOYuOR7JJzemQ8BzT6X42KPAAJxySz04UrTtxWkIPfX1OMwjlJB3RL932H+Ovv5KtU96VG8Z9npPK8li5uWYFLTwesH86sT1Kuah5Ct46D9d1LpZUzFt+sntGAQ5RY90QDItbr2OvkvPQ2kKjQpAsPb80x60yyhzWsJ3kZVejIuxv6pM7KF0hF1IHjj8YGRFtZxZx8NyYCoZaSUZQ+vl4djyHpvnKViNcXckgsMlJtQKfeluY1lOjGvciLctP/6rVKbOjGeENZc8TcwpnCZtmmBgrzW8wnY2/7f8q/kMILNqfu5YXTFQRYoJ3Nn7NP8oWlZ/hJN+Zt0ywIcxTswwzNfEM2/H8A4dSp8YZlI0kf09nAsLAgPQmaRcZjl8INzmlHEfy8vD+ooNDoPt7m7DlPpcNPKWzwjQ=
+deploy:
+  provider: rubygems
+  api_key:
+    secure: ggaLaFKzqIR4B8ofCVwEXAdEwgnvzDx++UTRS6hVNsLVA6QQxl4EmZvOhX/Dfv/M/CaeWSB5N04Rlo4h95KkCmCh4UHH9aCK/N5CK2NpbVuyVF8JVOiCaqEvGkPcQ6kHZE53E1w+xU6ag/I97ZXxwO0biSCiTgeFXXKPotXtBbDk2Kr3CPJ7zw5ry7n3ZBWeqrkoVzatBBLL+SONSnDvkL/rqVqffdYmyDz4Y/8tcn/Suk+FesARid4/J6DPgINgE4jNBu9VLwwb29QyN0hJAsE1csbEaOGXoI8xm444PF2HHlZdlfkpNf0ecX8h60mPqEKAztZLWT86asMrD337dpdVkWmQEgGU1bq4CGtJMqnGP4Yw+Qmip+jYaJu1Cc9/IKy/SzdtA4zzSjljrj3THxlnMMYILp50q9r+qyOPNoab3KutTis1ZOqD08pfQEfgsVVjmoSOFjRjbVMIVk89Si5pqehMdZKyV2n3KdEbyG/2MCsNDuBlV9ZaYShCMxSsDOo/x9vRwAP/E2Byud76crcoqGKS3if9oPj+bPPTjSLbYkSzoCYkus9TVp4sV43vgQ7FR9QwEY5oGQbSMdAXp65d43/SmSFRhBTqT6E4yfc18ivSyz6m0/ItHXTQHzHdfjfp5M2wnOWzWGO6oYM07L+XIWwUpzAI+KiXcRYe7PE=
+  gem: lxd-common
+  on:
+    tags: true
+    rvm: 2.4.2

data/Vagrantfile

@@ -4,11 +4,7 @@
 Vagrant.configure('2') do |config|
   config.vm.box = 'ubuntu/xenial64'
   # config.vm.box = 'ubuntu/trusty64'
-  # config.vm.provider 'virtualbox' do |vb|
-  #   vb.memory = '512'
-  # end
 
-  # apt-get install -y -t xenial-backports lxd lxd-client
   config.vm.provision 'chef_apply' do |chef|
     chef.recipe = File.read 'spec/provision_recipe.rb'
   end

data/lib/nexussw/lxd/driver/mixins/rest.rb

@@ -86,14 +86,19 @@ module NexusSW
 
           def delete_container(container_id)
             return unless container_exists? container_id
-            stop_container container_id,  force: true
-
-            # overcome a race condition where the host is busy doing 'something' async causing the delete command to pop an error
-            # https://github.com/lxc/lxd/issues/4063
-            # sleep 1
-
-            id = @hk.delete_container(container_id, sync: false)[:id]
-            @hk.wait_for_operation id
+            stop_container container_id, force: true
+
+            # ISSUE 17: something upstream is causing a double-tap on the REST endpoint
+            begin
+              @hk.delete_container container_id
+            rescue ::Faraday::ConnectionFailed, ::Hyperkit::BadRequest
+              LXD.with_timeout_and_retries timeout: 120 do
+                loop do
+                  return unless container_exists? container_id
+                  sleep 0.3
+                end
+              end
+            end
           end
 
           def container_status(container_id)
@@ -110,10 +115,7 @@ module NexusSW
           end
 
           def container_exists?(container_id)
-            return true if container_status(container_id)
-            return false
-          rescue
-            false
+            hk.containers.include? container_id
           end
 
           protected

data/lib/nexussw/lxd/transport/mixins/cli.rb

@@ -19,7 +19,7 @@ module NexusSW
 
           include Helpers::UploadFolder
 
-          def execute(command, options = {})
+          def execute(command, options = {}, &block)
             mycommand = command.is_a?(Array) ? command.join(' ') : command
             subcommand = options[:subcommand] || "exec #{container_name} --"
             mycommand = "lxc #{subcommand} #{mycommand}"
@@ -27,13 +27,13 @@ module NexusSW
             # We would have competing timeout logic depending on who the inner transport is
             # I'll just let rest & local do the timeouts, and if inner is a chef sourced transport, they have timeout logic of their own
             # with_timeout_and_retries(options) do
-            inner_transport.execute mycommand, options
+            inner_transport.execute mycommand, options, &block
           end
 
           def read_file(path)
             tfile = inner_mktmp
             retval = execute("#{@container_name}#{path} #{tfile}", subcommand: 'file pull', capture: false)
-            return '' if retval.exitstatus == 1
+            # return '' if retval.exitstatus == 1
             retval.error!
             return inner_transport.read_file tfile
           ensure
@@ -67,8 +67,8 @@ module NexusSW
           end
 
           def upload_folder(local_path, path)
-            return super unless config[:info] && config[:info][:api_extensions] && config[:info][:api_extensions].include?('directory_manipulation')
-            execute("-r #{localname} #{container_name}#{path}", subcommand: 'file push', capture: false).error!
+            return super unless config[:info] && config[:info]['api_extensions'] && config[:info]['api_extensions'].include?('directory_manipulation')
+            execute("-r #{local_path} #{container_name}#{path}", subcommand: 'file push', capture: false).error!
           end
 
           def add_remote(host_name)

data/lib/nexussw/lxd/transport/mixins/helpers/execute.rb

@@ -11,7 +11,7 @@ module NexusSW
                 @exitstatus = exitstatus
               end
 
-              attr_reader :exitstatus, :options, :command
+              attr_reader :options, :command, :exitstatus
 
               def stdout
                 options[:capture_options][:stdout] if options.key? :capture_options
@@ -24,8 +24,8 @@ module NexusSW
               def error!
                 return self if exitstatus == 0
                 msg = "Error: '#{command}' failed with exit code #{exitstatus}.\n"
-                msg += "STDOUT: #{stdout}" if stdout && !stdout.empty?
-                msg += "STDERR: #{stderr}" if stderr && !stderr.empty?
+                msg += "STDOUT: #{stdout}" if stdout.is_a?(String) && !stdout.empty?
+                msg += "STDERR: #{stderr}" if stderr.is_a?(String) && !stderr.empty?
                 raise msg
               end
             end
@@ -45,6 +45,31 @@ module NexusSW
 
               execute_chunked(command, options.merge(capture_options: capture_options), &capture_options[:capture])
             end
+
+            class InteractiveResult < ExecuteResult
+              def initialize(command, options, exitstatus, stdin, thread = nil)
+                super(command, options, exitstatus)
+                @stdin = stdin
+                @thread = thread
+              end
+
+              attr_reader :stdin, :thread
+              attr_accessor :exitstatus
+
+              def capture_output(&block)
+                @block = block if block_given?
+              end
+
+              def send_output(stdout_chunk)
+                return unless @block
+                @block.call stdout_chunk
+              end
+
+              def error!
+                thread.join if thread.respond_to? :join
+                super
+              end
+            end
           end
         end
       end

data/lib/nexussw/lxd/transport/mixins/helpers/upload_folder.rb

@@ -36,7 +36,7 @@ module NexusSW
                 # TODO: serious: make sure the tar extract does an overwrite of existing files
                 #   multiple converge support as well as CI cycle/dev updated files get updated instead of .1 suffixed (?)
                 #   I think I need a flag (it's been a while)
-                execute("bash -c 'mkdir -p #{path} && cd #{path} && tar -xf #{fname} && rm -rf #{fname}'", capture: false).error!
+                execute("bash -c 'mkdir -p #{path} && cd #{path} && tar -xf #{fname} && rm -rf #{fname}'").error!
               ensure
                 tfile.unlink
               end

data/lib/nexussw/lxd/transport/mixins/local.rb

@@ -15,37 +15,34 @@ module NexusSW
 
           include Helpers::ExecuteMixin
 
-          def execute_chunked(command, options)
+          def execute_chunked(command, options, &block)
             NIO::WebSocket::Reactor.start
             LXD.with_timeout_and_retries options do
-              # Let's borrow the NIO::WebSocket reactor
-              Open3.popen3(command) do |_stdin, stdout, stderr, th|
-                mon_out = mon_err = nil
-                NIO::WebSocket::Reactor.queue_task do
-                  mon_out = NIO::WebSocket::Reactor.selector.register(stdout, :r)
-                  mon_out.value = proc do
-                    data = read(mon_out) # read regardless of block_given? so that we don't spin out on :r availability
-                    yield(data) if data && block_given?
+              Open3.popen3(command) do |stdin, stdout, stderr, th|
+                if options[:capture] == :interactive
+                  # return immediately if interactive so that stdin may be used
+                  return Helpers::ExecuteMixin::InteractiveResult.new(command, options, -1, stdin, th).tap do |active|
+                    chunk_callback(stdout, stderr) do |stdout_chunk, stderr_chunk|
+                      active.send_output stdout_chunk if stdout_chunk
+                      active.send_output stderr_chunk if stderr_chunk
+                    end
+                    yield active
+                    active.exitstatus = th.value.exitstatus
                   end
-                end
-                NIO::WebSocket::Reactor.queue_task do
-                  mon_err = NIO::WebSocket::Reactor.selector.register(stderr, :r)
-                  mon_err.value = proc do
-                    data = read(mon_err) # read regardless of block_given? so that we don't spin out on :r availability
-                    yield(nil, data) if data && block_given?
+                else
+                  chunk_callback(stdout, stderr, &block) if block_given?
+                  th.join
+                  loop do
+                    return Helpers::ExecuteMixin::ExecuteResult.new(command, options, th.value.exitstatus) if th.value.exited? && mon_out && mon_err && mon_out.closed? && mon_err.closed?
+                    Thread.pass
                   end
                 end
-                th.join
-                loop do
-                  return Helpers::ExecuteMixin::ExecuteResult.new(command, options, th.value.exitstatus) if th.value.exited? && mon_out && mon_err && mon_out.closed? && mon_err.closed?
-                  Thread.pass
-                end
               end
             end
           end
 
           def read_file(path)
-            return '' unless File.exist? path
+            # return '' unless File.exist? path
             File.read path
           end
 
@@ -57,6 +54,8 @@ module NexusSW
 
           private
 
+          attr_reader :mon_out, :mon_err
+
           def read(monitor)
             monitor.io.read_nonblock(16384)
           rescue IO::WaitReadable # rubocop:disable Lint/ShadowedException
@@ -65,6 +64,23 @@ module NexusSW
             monitor.close
             return nil
           end
+
+          def chunk_callback(stdout, stderr)
+            NIO::WebSocket::Reactor.queue_task do
+              @mon_out = NIO::WebSocket::Reactor.selector.register(stdout, :r)
+              @mon_out.value = proc do
+                data = read(@mon_out) # read regardless of block_given? so that we don't spin out on :r availability
+                yield(data) if data
+              end
+            end
+            NIO::WebSocket::Reactor.queue_task do
+              @mon_err = NIO::WebSocket::Reactor.selector.register(stderr, :r)
+              @mon_err.value = proc do
+                data = read(@mon_err) # read regardless of block_given? so that we don't spin out on :r availability
+                yield(nil, data) if data
+              end
+            end
+          end
         end
       end
     end

data/lib/nexussw/lxd/transport/mixins/rest.rb

@@ -22,20 +22,85 @@ module NexusSW
           include Helpers::ExecuteMixin
           include Helpers::UploadFolder
 
+          class StdinStub
+            # return self as an IO (un)like object
+            def initialize(driver)
+              @driver = driver
+            end
+            attr_reader :driver
+
+            # return a real IO object for parity with Local Transport
+            def self.pipe(driver)
+              NIO::WebSocket::Reactor.start
+              reader, writer = IO.pipe
+              NIO::WebSocket::Reactor.queue_task do
+                iomon = NIO::WebSocket::Reactor.selector.register(reader, :r)
+                iomon.value = proc do
+                  data = read(iomon)
+                  driver.binary data if data
+                end
+              end
+              writer
+            end
+
+            def write(data)
+              driver.binary data
+            end
+
+            def self.read(monitor)
+              monitor.io.read_nonblock(16384)
+            rescue IO::WaitReadable # rubocop:disable Lint/ShadowedException
+              return nil
+            rescue Errno::ECONNRESET, EOFError, IOError
+              monitor.close
+              return nil
+            end
+          end
+
           def execute_chunked(command, options = {}, &block)
             opid = nil
             backchannel = nil
-            if block_given? # Allow for an optimized case that doesn't require the support of 3 new websocket connections
-              retval = hk.execute_command(container_name, command, wait_for_websocket: true, interactive: false, sync: false)
+            getlogs = false
+            if block_given? && (options[:capture] || !config[:info][:api_extensions].include?('container_exec_recording'))
+              hkopts = { wait_for_websocket: true, interactive: false, sync: false }
+              hkopts[:interactive] = true if options[:capture] == :interactive
+              retval = hk.execute_command(container_name, command, hkopts)
+              opid = retval[:id]
+              backchannel = options[:capture] == :interactive ? ws_connect(opid, retval[:metadata][:fds]) : ws_connect(opid, retval[:metadata][:fds], &block)
+
+              # patch for interactive session
+              return Helpers::ExecuteMixin::InteractiveResult.new(command, options, -1, StdinStub.pipe(backchannel.waitlist[:'0']), backchannel).tap do |active|
+                backchannel.callback = proc do |stdout|
+                  active.send_output stdout
+                end
+                yield active
+                backchannel.exit if backchannel.respond_to? :exit
+                retval = hk.wait_for_operation opid
+                active.exitstatus = retval[:metadata][:return].to_i
+              end if options[:capture] == :interactive
+            elsif block_given? && config[:info][:api_extensions].include?('container_exec_recording')
+              getlogs = true
+              retval = hk.execute_command(container_name, command, record_output: true, interactive: false, sync: false)
               opid = retval[:id]
-              backchannel = ws_connect opid, retval[:metadata][:fds], &block
             else
               opid = hk.execute_command(container_name, command, sync: false)[:id]
             end
             LXD.with_timeout_and_retries({ timeout: 0 }.merge(options)) do
               begin
                 retval = hk.wait_for_operation opid
-                backchannel.exit if backchannel.respond_to? :exit
+                backchannel.join if backchannel.respond_to? :join
+                if getlogs
+                  begin
+                    stdout_log = retval[:metadata][:output][:'1'].split('/').last
+                    stderr_log = retval[:metadata][:output][:'2'].split('/').last
+                    stdout = hk.log container_name, stdout_log
+                    stderr = hk.log container_name, stderr_log
+                    yield stdout, stderr
+                  ensure
+                    hk.delete_log container_name, stdout_log
+                    hk.delete_log container_name, stderr_log
+                  end
+                end
                 return Helpers::ExecuteMixin::ExecuteResult.new command, options, retval[:metadata][:return].to_i
               rescue Faraday::TimeoutError => e
                 raise Timeout::Retry.new e # rubocop:disable Style/RaiseArgs
@@ -43,10 +108,11 @@ module NexusSW
             end
           end
 
+          # '' instead of Hyperkit::NotFound is a chef-provisioning expectation - at this level we'll let the exception propagate
           def read_file(path)
             hk.read_file container_name, path
-          rescue ::Hyperkit::NotFound
-            return ''
+            # rescue ::Hyperkit::NotFound
+            #   return ''
           end
 
           def write_file(path, content)
@@ -64,16 +130,51 @@ module NexusSW
 
           protected
 
-          class WSWrapper
-            def initialize(waitlist)
-              @waitlist = waitlist.compact
+          class WSController
+            def initialize(ws_options, baseurl, endpoints, &block)
+              @waitlist = {}
+              @callback = block if block_given?
+              waitlist[:control] = NIO::WebSocket.connect(baseurl + endpoints[:control], ws_options) do |driver|
+                driver.on :io_error do # usually I get an EOF
+                  waitlist.each { |_, v| v.close if v.respond_to? :close }
+                end
+                driver.on :close do # but on occasion I get a legit close
+                  waitlist.each { |_, v| v.close if v.respond_to? :close }
+                end
+              end
+              waitlist[:'2'] = NIO::WebSocket.connect(baseurl + endpoints[:'2'], ws_options) do |driver|
+                driver.on :message do |ev|
+                  data = ev.data.is_a?(String) ? ev.data : ev.data.pack('U*')
+                  callback.call nil, data
+                end
+              end if endpoints[:'2']
+              waitlist[:'1'] = NIO::WebSocket.connect(baseurl + endpoints[:'1'], ws_options) do |driver|
+                driver.on :message do |ev|
+                  data = ev.data.is_a?(String) ? ev.data : ev.data.pack('U*')
+                  callback.call data
+                end
+              end if endpoints[:'1']
+              waitlist[:'0'] = NIO::WebSocket.connect(baseurl + endpoints[:'0'], ws_options) do |driver|
+                driver.on :message do |ev|
+                  data = ev.data.is_a?(String) ? ev.data : ev.data.pack('U*')
+                  callback.call data
+                end
+              end
             end
+
             attr_reader :waitlist
+            attr_accessor :callback
 
             def exit
+              waitlist.each do |_fd, driver|
+                driver.close
+              end
+            end
+
+            def join
               loop do
                 allclosed = true
-                waitlist.each do |driver|
+                waitlist.each do |_fd, driver|
                   allclosed = false unless driver.state == :closed
                 end
                 break if allclosed
@@ -83,7 +184,7 @@ module NexusSW
             end
           end
 
-          def ws_connect(opid, endpoints)
+          def ws_connect(opid, endpoints, &block)
             # NIO::WebSocket.log_traffic = true
             verify_ssl = OpenSSL::SSL::VERIFY_NONE if @driver_options[:verify_ssl] == false
             ws_options = { ssl_context: { verify_mode: verify_ssl } } unless verify_ssl.nil?
@@ -92,31 +193,7 @@ module NexusSW
             baseurl += '/' unless baseurl.end_with? '/'
             baseurl += "1.0/operations/#{opid}/websocket?secret="
 
-            pipes = {}
-            NIO::WebSocket.connect(baseurl + endpoints[:control], ws_options) do |driver|
-              driver.on :io_error do # usually I get an EOF
-                pipes.each { |_, v| v.close if v.respond_to? :close }
-              end
-              driver.on :close do # but on occasion I get a legit close
-                pipes.each { |_, v| v.close if v.respond_to? :close }
-              end
-            end
-            pipes[:'1'] = NIO::WebSocket.connect(baseurl + endpoints[:'1'], ws_options) do |driver|
-              driver.on :message do |ev|
-                data = ev.data.is_a?(String) ? ev.data : ev.data.pack('U*')
-                yield data
-              end
-            end
-            endpoints.each do |fd, secret|
-              next if [:control, :'1'].include? fd
-              pipes[fd] = NIO::WebSocket.connect(baseurl + secret, ws_options) do |driver|
-                driver.on :message do |ev|
-                  data = ev.data.is_a?(String) ? ev.data : ev.data.pack('U*')
-                  yield nil, data
-                end
-              end
-            end
-            WSWrapper.new [pipes[:'1'], pipes[:'2']]
+            WSController.new ws_options, baseurl, endpoints, &block
           end
         end
       end

data/lib/nexussw/lxd/version.rb

@@ -1,5 +1,5 @@
 module NexusSW
   module LXD
-    VERSION = '0.5.0'.freeze
+    VERSION = '0.6.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lxd-common
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Sean Zachariasen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-16 00:00:00.000000000 Z
+date: 2017-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hyperkit
@@ -148,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.4
 signing_key: 
 specification_version: 4
 summary: Shared LXD Container Access Library