lxd-common 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4232e85c055f7dc0ca02c732241ec3045cd71291ab9cd3a2b17c85698a1c8002
-  data.tar.gz: 8ff20ac3b41f2a07312cf7e9fd3fa542fdd23cc74bb6821551fea9511b5e07f3
+  metadata.gz: 032aa7f8a09b027c6254b6e42c5c36328a0325804780ed7a6db13ec4f905d197
+  data.tar.gz: b52cf26ede270ceb806dfd552d97a6ef0b5693e10a98943d2d6f4e708ed0ca2f
 SHA512:
-  metadata.gz: 9b7706d7598d89e2b5e1b08863b12ecbe3b8404db520d61b2cba1acffc41ec7a217cd8f31a0c760548285428ef580eec3bd085847a065088eeab3072e1da5719
-  data.tar.gz: 5c1e444616de2a81b74c08ec5b57d4daced4a11a4875e2c3008488133ddb872758ef3c4ff4edca6348dfa547e4dfffafaf9d1d50744e9fe0bd716390dc5b11ab
+  metadata.gz: ae77de9d2025c2cdba5cd71740fec1766486520844b29f4dc11ebb786f69da90310033c2c722bdfb0db0234c19737b4c7c868cc3000b05a824caf12500ba9f15
+  data.tar.gz: 99dbeed1e5731e1050197a244c16c91215ba41b11c78d555d2ed16cb368b1cde11ff4f63975b6aedb83e38bb345df66c41f0b88fdd3e58687e63effb69868097

data/Gemfile

@@ -1,4 +1,3 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in lxd-common.gemspec
-gemspec
+source 'https://rubygems.org'
+
+gemspec

data/Rakefile

@@ -1,9 +1,10 @@
-require 'bundler/gem_tasks'
-require 'rspec/core/rake_task'
-
-RSpec::Core::RakeTask.new(:spec)
-RSpec::Core::RakeTask.new(:mock) do |task|
-  task.pattern = 'spec/**{,/*/**}/*_mock.rb'
-end
-
-task default: :spec
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:mock) do |task|
+  task.pattern = 'spec/**{,/*/**}/*_mock.rb'
+end
+
+task default: :spec

data/lib/nexussw/lxd/rest_api.rb

@@ -92,9 +92,7 @@ module NexusSW
       end
 
       def wait_for_operation(operation_id)
-        get "/1.0/operations/#{operation_id}/wait" do |response|
-          LXD.symbolize_keys(JSON.parse(response.body))
-        end
+        get "/1.0/operations/#{operation_id}/wait"
       end
 
       private

data/lib/nexussw/lxd/rest_api/connection.rb

@@ -35,8 +35,8 @@ module NexusSW
             url: baseurl,
             ssl: {
               verify: verify_ssl,
-              client_cert: OpenSSL::X509::Certificate.new(cert),
-              client_key: OpenSSL::PKey::RSA.new(key),
+              client_cert: client_cert,
+              client_key: client_key,
             },
           }
 
@@ -51,12 +51,12 @@ module NexusSW
           api_options[:ssl] || {}
         end
 
-        def cert
-          File.read(ssl_opts[:client_cert] || "#{ENV['HOME']}/.config/lxc/client.crt")
+        def client_cert
+          @client_cert ||= OpenSSL::X509::Certificate.new(File.read(ssl_opts[:client_cert] || "#{ENV['HOME']}/.config/lxc/client.crt"))
         end
 
-        def key
-          File.read(ssl_opts[:client_key] || "#{ENV['HOME']}/.config/lxc/client.key")
+        def client_key
+          @client_key ||= OpenSSL::PKey::RSA.new(File.read(ssl_opts[:client_key] || "#{ENV['HOME']}/.config/lxc/client.key"))
         end
 
         def verify_ssl
@@ -69,6 +69,7 @@ module NexusSW
         end
 
         def send_request(verb, relative_url, content = nil)
+          fileop = false
           response = connection.send(verb) do |req|
             req.url relative_url
             if content.is_a? Hash
@@ -77,6 +78,7 @@ module NexusSW
             elsif content # Only upon file upload at this time
               yield req if block_given?
               req.body = content.to_s
+              fileop = true
             end
           end
           if response.status >= 400
@@ -87,7 +89,7 @@ module NexusSW
             else raise RestAPI::Error, "Error #{err['error_code']}: #{err['error']}"
             end
           end
-          block_given? ? yield(response) : parse_response(response)
+          block_given? && !fileop ? yield(response) : parse_response(response)
         end
       end
     end

data/lib/nexussw/lxd/transport.rb

@@ -1,4 +1,5 @@
 require 'nexussw/lxd'
+require 'tempfile'
 
 module NexusSW
   module LXD
@@ -23,6 +24,10 @@ module NexusSW
         raise "#{self.class}#download_file not implemented"
       end
 
+      def download_folder(_path, _local_path)
+        raise "#{self.class}#download_folder not implemented"
+      end
+
       def upload_file(_local_path, _path, _options = {})
         raise "#{self.class}#upload_file not implemented"
       end
@@ -30,6 +35,21 @@ module NexusSW
       def upload_folder(_local_path, _path, _options = {})
         raise "#{self.class}#upload_folder not implemented"
       end
+
+      # kludge for windows environment
+      def self.remote_tempname(basename)
+        tfile = Tempfile.new(basename)
+        "/tmp/#{File.basename tfile.path}"
+      ensure
+        tfile.unlink
+      end
+
+      def self.local_tempdir
+        return ENV['TEMP'] unless !ENV['TEMP'] || ENV['TEMP'].empty?
+        return ENV['TMP'] unless !ENV['TMP'] || ENV['TMP'].empty?
+        return ENV['TMPDIR'] unless !ENV['TMPDIR'] || ENV['TMPDIR'].empty?
+        '/tmp'
+      end
     end
   end
 end

data/lib/nexussw/lxd/transport/mixins/cli.rb

@@ -1,6 +1,6 @@
 require 'nexussw/lxd/transport/mixins/local'
 require 'nexussw/lxd/transport/mixins/helpers/users'
-require 'nexussw/lxd/transport/mixins/helpers/upload_folder'
+require 'nexussw/lxd/transport/mixins/helpers/folder_txfr'
 require 'tempfile'
 require 'shellwords'
 
@@ -18,7 +18,7 @@ module NexusSW
 
           attr_reader :inner_transport, :punt, :container_name, :config
 
-          include Helpers::UploadFolder
+          include Helpers::FolderTxfr
           include Helpers::UsersMixin
 
           def execute(command, options = {}, &block)
@@ -33,7 +33,7 @@ module NexusSW
           end
 
           def read_file(path)
-            tfile = inner_mktmp
+            tfile = Transport.remote_tempname(container_name)
             retval = execute("#{@container_name}#{path} #{tfile}", subcommand: 'file pull', capture: false)
             # return '' if retval.exitstatus == 1
             retval.error!
@@ -45,7 +45,7 @@ module NexusSW
           def write_file(path, content, options = {})
             perms = file_perms(options)
 
-            tfile = inner_mktmp
+            tfile = Transport.remote_tempname(container_name)
             inner_transport.write_file tfile, content
             execute("#{tfile} #{container_name}#{path}", subcommand: "file push#{perms}", capture: false).error!
           ensure
@@ -53,9 +53,9 @@ module NexusSW
           end
 
           def download_file(path, local_path)
-            tfile = inner_mktmp if punt
+            tfile = Transport.remote_tempname(container_name) if punt
             localname = tfile || local_path
-            execute("#{container_name}#{path} #{localname}", subcommand: 'file pull', capture: false).error!
+            execute("#{container_name}#{path} #{localname}", subcommand: 'file pull').error!
             inner_transport.download_file tfile, local_path if tfile
           ensure
             inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
@@ -64,10 +64,10 @@ module NexusSW
           def upload_file(local_path, path, options = {})
             perms = file_perms(options)
 
-            tfile = inner_mktmp if punt
+            tfile = Transport.remote_tempname(container_name) if punt
             localname = tfile || local_path
             inner_transport.upload_file local_path, tfile if tfile
-            execute("#{localname} #{container_name}#{path}", subcommand: "file push#{perms}", capture: false).error!
+            execute("#{localname} #{container_name}#{path}", subcommand: "file push#{perms}").error!
           ensure
             inner_transport.execute("rm -rf #{tfile}", capture: false) if tfile
           end
@@ -79,6 +79,12 @@ module NexusSW
             execute("-r #{local_path} #{container_name}#{path}", subcommand: "file push#{perms}", capture: false).error!
           end
 
+          def download_folder(path, local_path)
+            return super unless config[:info] && config[:info]['api_extensions'] && config[:info]['api_extensions'].include?('directory_manipulation')
+
+            execute("-r #{container_name}#{path} #{local_path}", subcommand: 'file pull', capture: false).error!
+          end
+
           def add_remote(host_name)
             execute("add #{host_name} --accept-certificate", subcommand: 'remote').error! unless remote? host_name
           end
@@ -101,14 +107,6 @@ module NexusSW
 
           private
 
-          # kludge for windows environment
-          def inner_mktmp
-            tfile = Tempfile.new(container_name)
-            "/tmp/#{File.basename tfile.path}"
-          ensure
-            tfile.unlink
-          end
-
           def file_perms(options = {})
             perms = ''
             perms += " --uid=#{options[:uid] || uid || 0}"

data/lib/nexussw/lxd/transport/mixins/helpers/{upload_folder.rb → folder_txfr.rb}

@@ -1,41 +1,69 @@
+require 'zlib'
+require 'archive/tar/minitar'
+
 module NexusSW
   module LXD
     class Transport
       module Mixins
         module Helpers
-          module UploadFolder
+          module FolderTxfr
             def upload_folder(local_path, path, options = {})
               upload_using_tarball(local_path, path, options) || upload_files_individually(local_path, path, options)
             end
 
+            def download_folder(path, local_path)
+              download_using_tarball(path, local_path) || download_files_individually(path, local_path)
+            end
+
             def upload_files_individually(local_path, path, options = {})
+              dest = File.join(path, File.basename(local_path))
+              execute('mkdir -p ' + dest).error! # for parity with tarball extract
               Dir.entries(local_path).map { |f| (f == '.' || f == '..') ? nil : File.join(local_path, f) }.compact.each do |f|
-                dest = File.join(path, File.basename(local_path))
                 upload_files_individually f, dest, options if File.directory? f
                 upload_file f, File.join(dest, File.basename(f)), options if File.file? f
               end
             end
 
+            def download_files_individually(path, local_path)
+              dest = File.join(local_path, File.basename(path))
+              execute("bash -c 'cd #{path} && find -type d'").error!.stdout.each_line do |line|
+                newdir = line.strip.sub(/^\./, dest)
+                Dir.mkdir newdir unless Dir.exist? newdir
+              end
+              execute("bash -c 'cd #{path} && find ! -type d'").error!.stdout.each_line do |line|
+                download_file line.strip.sub(/^\./, path), line.strip.sub(/^\./, dest)
+              end
+            end
+
+            # gzip(-z) or bzip2(-j) (these are the only 2 on trusty atm)
+            def download_using_tarball(path, local_path)
+              return false unless can_archive?
+              tfile = Transport.remote_tempname(container_name)
+              tarball_name = File.join Transport.local_tempdir, File.basename(tfile) + '.tgz'
+              execute("tar -czf #{tfile} -C #{File.dirname path} #{File.basename path}").error!
+
+              download_file tfile, tarball_name
+
+              Archive::Tar::Minitar.unpack Zlib::GzipReader.new(File.open(tarball_name, 'rb')), local_path
+            ensure
+              if tarball_name
+                File.delete tarball_name if File.exist? tarball_name
+                execute "rm -rf #{tfile}"
+              end
+            end
+
             def upload_using_tarball(local_path, path, options = {})
               return false unless can_archive?
-              # TODO: should I return false upon error?  i.e. retry with individual file uploads if this fails?
-              #   lets see how this does in the wild before deciding
-              flag, ext = compression
               begin
                 tfile = Tempfile.new(container_name)
                 tfile.close
-                `tar -c#{flag}f #{tfile.path} -C #{File.dirname local_path} ./#{File.basename local_path}`
-                # on that above note we'll do this at least
-                # raise "Unable to create archive #{tfile.path}" if File.zero? tfile.path
-                if File.zero? tfile.path
-                  @can_archive = false
-                  return false
+                Dir.chdir File.dirname(local_path) do
+                  Archive::Tar::Minitar.pack File.basename(local_path), Zlib::GzipWriter.new(File.open(tfile.path, 'wb'))
                 end
-                fname = '/tmp/' + File.basename(tfile.path) + ".tar#{ext}"
+                # `tar -c#{flag}f #{tfile.path} -C #{File.dirname local_path} ./#{File.basename local_path}`
+                fname = '/tmp/' + File.basename(tfile.path) + '.tgz'
                 upload_file tfile.path, fname, options
-                # TODO: serious: make sure the tar extract does an overwrite of existing files
-                #   multiple converge support as well as CI cycle/dev updated files get updated instead of .1 suffixed (?)
-                #   I think I need a flag (it's been a while)
+
                 myuid = options[:uid] || uid || (0 if is_a?(Mixins::CLI))
                 mygid = options[:gid] || gid || (0 if is_a?(Mixins::CLI))
                 mymode = options[:file_mode] || file_mode
@@ -57,25 +85,11 @@ module NexusSW
                                   return false if respond_to?(:inner_transport) && inner_transport.respond_to?(:mock)
                                   return false if respond_to?(:inner_transport) && inner_transport.respond_to?(:inner_transport) && inner_transport.inner_transport.respond_to?(:mock)
                                   return false if respond_to?(:inner_transport) && inner_transport.respond_to?(:api) && inner_transport.api.respond_to?(:mock)
-                                  `tar --version`
                                   true
                                 rescue
                                   false
                                 end
             end
-
-            # gzip(-z) or bzip2(-j) (these are the only 2 on trusty atm)
-            def compression
-              @compression ||= begin
-                                  which = execute('bash -c "which gzip || which bzip2 || true"').stdout.strip
-                                  which = File.basename(which) if which
-                                  case which
-                                  when 'gzip' then ['z', '.gz']
-                                  when 'bzip2' then ['j', '.bzip2']
-                                  else ['', '']
-                                  end
-                                end
-            end
           end
         end
       end

data/lib/nexussw/lxd/transport/mixins/rest.rb

@@ -1,6 +1,6 @@
 require 'nexussw/lxd/transport/mixins/helpers/execute'
 require 'nexussw/lxd/transport/mixins/helpers/users'
-require 'nexussw/lxd/transport/mixins/helpers/upload_folder'
+require 'nexussw/lxd/transport/mixins/helpers/folder_txfr'
 require 'nio/websocket'
 require 'tempfile'
 require 'json'
@@ -23,7 +23,7 @@ module NexusSW
           attr_reader :api, :rest_endpoint, :container_name, :config
 
           include Helpers::ExecuteMixin
-          include Helpers::UploadFolder
+          include Helpers::FolderTxfr
           include Helpers::UsersMixin
 
           class StdinStub

data/lib/nexussw/lxd/version.rb

@@ -1,5 +1,5 @@
 module NexusSW
   module LXD
-    VERSION = '0.8.1'.freeze
+    VERSION = '0.9.0'.freeze
   end
 end

data/lxd-common.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'faraday', '~> 0.13'
   spec.add_dependency 'nio4r-websocket', '~> 0.6'
+  spec.add_dependency 'minitar', '~> 0.5'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lxd-common
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Sean Zachariasen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-21 00:00:00.000000000 Z
+date: 2018-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: minitar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -126,7 +140,7 @@ files:
 - lib/nexussw/lxd/transport/local.rb
 - lib/nexussw/lxd/transport/mixins/cli.rb
 - lib/nexussw/lxd/transport/mixins/helpers/execute.rb
-- lib/nexussw/lxd/transport/mixins/helpers/upload_folder.rb
+- lib/nexussw/lxd/transport/mixins/helpers/folder_txfr.rb
 - lib/nexussw/lxd/transport/mixins/helpers/users.rb
 - lib/nexussw/lxd/transport/mixins/local.rb
 - lib/nexussw/lxd/transport/mixins/rest.rb