lxd-common 0.4.1

data/lib/nexussw/lxd/driver/cli.rb

@@ -0,0 +1,12 @@
+require 'nexussw/lxd/driver'
+require 'nexussw/lxd/driver/mixins/cli'
+
+module NexusSW
+  module LXD
+    class Driver
+      class CLI < Driver
+        include Mixins::CLI
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/driver/mixins/cli.rb

@@ -0,0 +1,182 @@
+require 'tempfile'
+require 'yaml'
+require 'json'
+
+module NexusSW
+  module LXD
+    class Driver
+      module Mixins
+        module CLI
+          def initialize(inner_transport, driver_options = {})
+            @inner_transport = inner_transport
+            @driver_options = driver_options || {}
+          end
+
+          attr_reader :inner_transport, :driver_options
+
+          def create_container(container_name, container_options = {})
+            if container_exists? container_name
+              start_container container_name # Start for Parity with the below logic (`lxc launch` auto starts)
+              return container_name
+            end
+            cline = "lxc launch #{image_alias(container_options)} #{container_name}"
+            profiles = container_options[:profiles] || []
+            profiles.each { |p| cline += " -p #{p}" }
+            configs = container_options[:config] || {}
+            configs.each { |k, v| cline += " -c #{k}=#{v}" }
+            inner_transport.execute(cline).error!
+            wait_for_status container_name, 'running'
+            container_name
+          end
+
+          def start_container(container_id)
+            return if container_status(container_id) == 'running'
+            inner_transport.execute("lxc start #{container_id}").error!
+            wait_for_status container_id, 'running'
+          end
+
+          def stop_container(container_id, options = {})
+            options ||= {} # default behavior: no timeout or retries.  These functions are up to the consumer's context and not really 'sane' defaults
+            return if container_status(container_id) == 'stopped'
+            return inner_transport.execute("lxc stop #{container_id} --force", capture: false).error! if options[:force]
+            LXD.with_timeout_and_retries(options) do
+              return if container_status(container_id) == 'stopped'
+              timeout = " --timeout=#{options[:retry_interval]}" if options[:retry_interval]
+              retval = inner_transport.execute("lxc stop #{container_id}#{timeout || ''}", capture: false)
+              begin
+                retval.error!
+              rescue => e
+                return if container_status(container_id) == 'stopped'
+                # can't distinguish between timeout, or other error.
+                # but if the status call is not popping a 404, and we're not stopped, then a retry is worth it
+                raise Timeout::Retry.new(e) if timeout # rubocop:disable Style/RaiseArgs
+                raise
+              end
+            end
+            wait_for_status container_id, 'stopped'
+          end
+
+          def delete_container(container_id)
+            return unless container_exists? container_id
+            inner_transport.execute("lxc delete #{container_id} --force", capture: false).error!
+          end
+
+          def container_status(container_id)
+            STATUS_CODES[container(container_id)[:status_code].to_i]
+          end
+
+          def convert_keys(oldhash)
+            return oldhash unless oldhash.is_a?(Hash) || oldhash.is_a?(Array)
+            retval = {}
+            if oldhash.is_a? Array
+              retval = []
+              oldhash.each { |v| retval << convert_keys(v) }
+            else
+              oldhash.each do |k, v|
+                retval[k.to_sym] = convert_keys(v)
+              end
+            end
+            retval
+          end
+
+          # YAML is not supported until somewhere in the feature branch
+          #   the YAML return has :state and :container at the root level
+          # the JSON return has no :container (:container is root)
+          #   and has :state underneath that
+          # (CLI Only) and :state is only available if the container is running
+          def container_state(container_id)
+            res = inner_transport.execute("lxc list #{container_id} --format=json")
+            res.error!
+            JSON.parse(res.stdout).each do |c|
+              return convert_keys(c['state']) if c['name'] == container_id
+            end
+            nil
+          end
+
+          def container(container_id)
+            res = inner_transport.execute("lxc list #{container_id} --format=json")
+            res.error!
+            JSON.parse(res.stdout).each do |c|
+              return convert_keys(c.except('state')) if c['name'] == container_id
+            end
+            nil
+          end
+
+          def container_exists?(container_id)
+            return true if container_status(container_id)
+            return false
+          rescue
+            false
+          end
+
+          include WaitMixin
+
+          protected
+
+          def wait_for_status(container_id, newstatus)
+            loop do
+              status = container_status(container_id)
+              return if status == newstatus
+              NIO::WebSocket.logger.debug "#{container_id} status = '#{status}'.  Waiting for '#{newstatus}'"
+              sleep 0.5
+            end
+          end
+
+          private
+
+          def remote_for!(url, protocol = 'lxd')
+            raise 'Protocol is required' unless protocol # protect me from accidentally slipping in a nil
+            # normalize the url and 'require' protocol to protect against a scenario:
+            #   1) user only specifies https://someimageserver.org without specifying the protocol
+            #   2) the rest of this function would blindly add that without saying the protocol
+            #   3) 'lxc remote add' would add that remote, but defaults to the lxd protocol and appends ':8443' to the saved url
+            #   4) the next time this function is called we would not match that same entry due to the ':8443'
+            #   5) ultimately resulting in us adding a new remote EVERY time this function is called
+            port = url.split(':', 3)[2]
+            url += ':8443' unless port || protocol != 'lxd'
+            remotes = begin
+                        YAML.load(inner_transport.read_file('~/.config/lxc/config.yml')) || {}
+                      rescue
+                        {}
+                      end
+            # make sure these default entries are available to us even if config.yml isn't created yet
+            # and i've seen instances where these defaults don't live in the config.yml
+            remotes = { 'remotes' => {
+              'images' => { 'addr' => 'https://images.linuxcontainers.org' },
+              'ubuntu' => { 'addr' => 'https://cloud-images.ubuntu.com/releases' },
+              'ubuntu-daily' => { 'addr' => 'https://cloud-images.ubuntu.com/daily' },
+            } }.merge remotes
+            max = 0
+            remotes['remotes'].each do |remote, data|
+              return remote.to_s if data['addr'] == url
+              num = remote.to_s.split('-', 2)[1] if remote.to_s.start_with? 'images-'
+              max = num.to_i if num && num.to_i > max
+            end
+            remote = "images-#{max + 1}"
+            inner_transport.execute("lxc remote add #{remote} #{url} --accept-certificate --protocol=#{protocol}").error!
+            remote
+          end
+
+          def image(properties, remote = '')
+            return nil unless properties && properties.any?
+            cline = "lxc image list #{remote} --format=json"
+            properties.each { |k, v| cline += " #{k}=#{v}" }
+            res = inner_transport.execute cline
+            res.error!
+            res = JSON.parse(res.stdout)
+            return res[0]['fingerprint'] if res.any?
+          end
+
+          def image_alias(container_options)
+            remote = container_options[:server] ? remote_for!(container_options[:server], container_options[:protocol] || 'lxd') + ':' : ''
+            name = container_options[:alias]
+            name ||= container_options[:fingerprint]
+            name ||= image(container_options[:properties], remote)
+            raise 'No image parameters.  One of alias, fingerprint, or properties must be specified (The CLI interface does not support empty containers)' unless name
+            "#{remote}#{name}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/driver/mixins/rest.rb

@@ -0,0 +1,128 @@
+require 'hyperkit'
+
+module NexusSW
+  module LXD
+    class Driver
+      module Mixins
+        module Rest
+          # PARITY note: CLI functions are on an indefinite timeout by default, yet we have a 2 minute socket read timeout
+          # Leaving it alone, for now, on calls that are quick in nature
+          # Adapting on known long running calls such as create, stop, execute
+          # REQUEST_TIMEOUT = 120 # upstream default: 120
+          def initialize(rest_endpoint, driver_options = {}, inner_driver = nil)
+            @rest_endpoint = rest_endpoint
+            @driver_options = driver_options
+            hkoptions = (driver_options || {}).merge(
+              api_endpoint: rest_endpoint,
+              auto_sync: true
+            )
+            @hk = inner_driver || Hyperkit::Client.new(hkoptions)
+            # HACK: can't otherwise get at the request timeout because sawyer is in the way
+            # Beware of unused function in hyperkit: reset_agent  If that gets used it'll undo this timeout
+            # unneeded while default valued: @hk.agent.instance_variable_get(:@conn).options[:timeout] = REQUEST_TIMEOUT
+          end
+
+          attr_reader :hk, :rest_endpoint, :driver_options
+
+          def create_container(container_name, container_options = {})
+            if container_exists?(container_name)
+              start_container container_name # Start the container for Parity with the CLI
+              return container_name
+            end
+            # we'll break this apart and time it out for those with slow net (and this was my 3 minute stress test case with good net)
+            # parity note: CLI will run indefinitely rather than timeout hence the 0 timeout
+            retry_forever do
+              @hk.create_container(container_name, container_options.merge(sync: false))
+            end
+            start_container container_name
+            container_name
+          end
+
+          def start_container(container_id)
+            return if container_status(container_id) == 'running'
+            retry_forever do
+              @hk.start_container(container_id, sync: false)
+            end
+            wait_for_status container_id, 'running'
+          end
+
+          def stop_container(container_id, options = {})
+            return if container_status(container_id) == 'stopped'
+            return @hk.stop_container(container_id, force: true) if options[:force]
+            last_id = nil
+            use_last = false
+            LXD.with_timeout_and_retries({ timeout: 0 }.merge(options)) do # timeout: 0 to enable retry functionality
+              return if container_status(container_id) == 'stopped'
+              begin
+                unless use_last
+                  # Keep resubmitting until the server complains (Stops will be ignored/hang if init is not yet listening for SIGPWR i.e. recently started)
+                  begin
+                    last_id = @hk.stop_container(container_id, sync: false)[:id] # TODO: this 'could' hang for 2 minutes?  and then we'd never get a last_id if that's where the hang happens, and then it 'could' error on retry
+                  rescue Hyperkit::BadRequest # Happens if a stop command has previously been accepted as well as other reasons.  handle that on next line
+                    raise unless last_id # if we have a last_id then a prior stop command has successfully initiated so we'll just wait on that one
+                    use_last = true
+                  end
+                end
+                @hk.wait_for_operation last_id # , options[:retry_interval]
+              rescue Faraday::TimeoutError => e
+                return if container_status(container_id) == 'stopped'
+                raise Timeout::Retry.new e # if options[:retry_interval] # rubocop:disable Style/RaiseArgs
+              end
+            end
+            wait_for_status container_id, 'stopped'
+          end
+
+          def delete_container(container_id)
+            return unless container_exists? container_id
+            stop_container container_id, force: true
+            @hk.delete_container(container_id)
+          end
+
+          def container_status(container_id)
+            STATUS_CODES[container(container_id)[:status_code].to_i]
+          end
+
+          def container_state(container_id)
+            return nil unless container_status(container_id) == 'running' # Parity with CLI
+            @hk.container_state(container_id)
+          end
+
+          def container(container_id)
+            @hk.container container_id
+          end
+
+          def container_exists?(container_id)
+            return true if container_status(container_id)
+            return false
+          rescue
+            false
+          end
+
+          include WaitMixin
+
+          protected
+
+          def wait_for_status(container_id, newstatus)
+            loop do
+              return if container_status(container_id) == newstatus
+              sleep 0.5
+            end
+          end
+
+          private
+
+          def retry_forever
+            retval = yield
+            LXD.with_timeout_and_retries timeout: 0 do
+              begin
+                @hk.wait_for_operation retval[:id]
+              rescue Faraday::TimeoutError => e
+                raise Timeout::Retry.new e # rubocop:disable Style/RaiseArgs
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/driver/rest.rb

@@ -0,0 +1,12 @@
+require 'nexussw/lxd/driver'
+require 'nexussw/lxd/driver/mixins/rest'
+
+module NexusSW
+  module LXD
+    class Driver
+      class Rest < Driver
+        include Mixins::Rest
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/transport.rb

@@ -0,0 +1,73 @@
+require 'nexussw/lxd'
+
+module NexusSW
+  module LXD
+    class Transport
+      class LXDExecuteResult
+        def initialize(command, options, exitstatus)
+          @command = command
+          @options = options || {}
+          @exitstatus = exitstatus
+        end
+
+        attr_reader :exitstatus, :options, :command
+
+        def stdout
+          options[:capture_options][:stdout] if options.key? :capture_options
+        end
+
+        def stderr
+          options[:capture_options][:stderr] if options.key? :capture_options
+        end
+
+        def error!
+          return self if exitstatus == 0
+          msg = "Error: '#{command}' failed with exit code #{exitstatus}.\n"
+          msg += "STDOUT: #{stdout}" if stdout && !stdout.empty?
+          msg += "STDERR: #{stderr}" if stderr && !stderr.empty?
+          raise msg
+        end
+      end
+
+      module ExecuteMixin
+        def execute(command, options = {}, &block)
+          options ||= {}
+          return execute_chunked(command, options) if options[:capture] == false && !block_given?
+
+          capture_options = { stdout: '', stderr: '' }
+          capture_options[:capture] = block if block_given?
+          capture_options[:capture] ||= options[:capture] if options[:capture].respond_to? :call
+          # capture_options[:capture] ||= options[:stream] if options[:stream].respond_to? :call
+          capture_options[:capture] ||= proc do |stdout_chunk, stderr_chunk|
+            capture_options[:stdout] += stdout_chunk if stdout_chunk
+            capture_options[:stderr] += stderr_chunk if stderr_chunk
+          end
+
+          execute_chunked(command, options.merge(capture_options: capture_options), &capture_options[:capture])
+        end
+      end
+
+      def execute(_command, _options = {})
+        raise 'NexusSW::LXD::Transport.execute not implemented'
+      end
+
+      def read_file(_path)
+        raise 'NexusSW::LXD::Transport.read_file not implemented'
+      end
+
+      def write_file(_path, _content)
+        raise 'NexusSW::LXD::Transport.write_file not implemented'
+      end
+
+      def download_file(_path, _local_path)
+        raise 'NexusSW::LXD::Transport.download_file not implemented'
+      end
+
+      def upload_file(_local_path, _path)
+        raise 'NexusSW::LXD::Transport.upload_file not implemented'
+      end
+      # protected
+      # def execute_chunked(_command, _options = {})
+    end
+  end
+end

data/lib/nexussw/lxd/transport/cli.rb

@@ -0,0 +1,12 @@
+require 'nexussw/lxd/transport'
+require 'nexussw/lxd/transport/mixins/cli'
+
+module NexusSW
+  module LXD
+    class Transport
+      class CLI < Transport
+        include Mixins::CLI
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/transport/local.rb

@@ -0,0 +1,12 @@
+require 'nexussw/lxd/transport'
+require 'nexussw/lxd/transport/mixins/local'
+
+module NexusSW
+  module LXD
+    class Transport
+      class Local < Transport
+        include Mixins::Local
+      end
+    end
+  end
+end

data/lib/nexussw/lxd/transport/mixins/cli.rb

@@ -0,0 +1,98 @@
+require 'nexussw/lxd/transport/mixins/local'
+require 'tempfile'
+require 'pp'
+
+module NexusSW
+  module LXD
+    class Transport
+      module Mixins
+        module CLI
+          def initialize(remote_transport, container_name, config = {})
+            @container_name = container_name
+            @config = config
+            @inner_transport = remote_transport
+            @punt = !inner_transport.is_a?(::NexusSW::LXD::Transport::Mixins::Local)
+          end
+          attr_reader :inner_transport, :punt, :container_name, :config
+
+          def execute(command, options = {})
+            mycommand = command.is_a?(Array) ? command.join(' ') : command
+            subcommand = options[:subcommand] || "exec #{container_name} --"
+            mycommand = "lxc #{subcommand} #{mycommand}"
+            options = options.except :subcommand if options.key? :subcommand
+            # We would have competing timeout logic depending on who the inner transport is
+            # I'll just let rest & local do the timeouts, and if inner is a chef sourced transport, they have timeout logic of their own
+            # with_timeout_and_retries(options) do
+            inner_transport.execute mycommand, options
+          end
+
+          def read_file(path)
+            tfile = inner_mktmp
+            retval = execute("#{@container_name}#{path} #{tfile}", subcommand: 'file pull', capture: false)
+            return '' if retval.exitstatus == 1
+            retval.error!
+            return inner_transport.read_file tfile
+          ensure
+            inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
+          end
+
+          def write_file(path, content)
+            tfile = inner_mktmp
+            inner_transport.write_file tfile, content
+            execute("#{tfile} #{container_name}#{path}", subcommand: 'file push', capture: false).error!
+          ensure
+            inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
+          end
+
+          def download_file(path, local_path)
+            tfile = inner_mktmp if punt
+            localname = tfile || local_path
+            execute("#{container_name}#{path} #{localname}", subcommand: 'file pull', capture: false).error!
+            inner_transport.download_file tfile, local_path if tfile
+          ensure
+            inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
+          end
+
+          def upload_file(local_path, path)
+            tfile = inner_mktmp if punt
+            localname = tfile || local_path
+            inner_transport.upload_file local_path, tfile if tfile
+            execute("#{localname} #{container_name}#{path}", subcommand: 'file push', capture: false).error!
+          ensure
+            inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
+          end
+
+          def add_remote(host_name)
+            execute("add #{host_name} --accept-certificate", subcommand: 'remote').error! unless remote? host_name
+          end
+
+          def linked_transport(host_name)
+            linked = inner_transport.linked_transport(host_name) if inner_transport.is_a?(::NexusSW::LXD::Transport::CLI)
+            return linked if linked
+            return nil unless remote?(host_name)
+            new(driver, inner_transport, "#{host_name}:#{container_name}", config)
+          end
+
+          def remote?(host_name)
+            result = execute 'list', subcommand: 'remote'
+            result.error!
+            result.stdout.each_line do |line|
+              return true if line.start_with? "| #{host_name} "
+            end
+            false
+          end
+
+          private
+
+          # kludge for windows environment
+          def inner_mktmp
+            tfile = Tempfile.new(container_name)
+            "/tmp/#{File.basename tfile.path}"
+          ensure
+            tfile.unlink
+          end
+        end
+      end
+    end
+  end
+end