train 0.28.0 → 0.29.0

data/lib/train/file/remote.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+module Train
+  class File
+    class Remote < Train::File
+      def basename(suffix = nil, sep = '/')
+        fail 'Not yet supported: Suffix in file.basename' unless suffix.nil?
+        @basename ||= detect_filename(path, sep || '/')
+      end
+
+      def stat
+        return @stat if defined?(@stat)
+        @stat = Train::Extras::Stat.stat(@spath, @backend, @follow_symlink)
+      end
+
+      # helper methods provided to any implementing class
+      private
+
+      def detect_filename(path, sep)
+        idx = path.rindex(sep)
+        return path if idx.nil?
+        idx += 1
+        return detect_filename(path[0..-2], sep) if idx == path.length
+        path[idx..-1]
+      end
+    end
+  end
+end

data/lib/train/file/remote/aix.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+require 'train/file/remote/unix'
+
+module Train
+  class File
+    class Remote
+      class Aix < Train::File::Remote::Unix
+        def link_path
+          return nil unless symlink?
+          @link_path ||=
+            @backend.run_command("perl -e 'print readlink shift' #{@spath}").stdout.chomp
+        end
+
+        def mounted
+          @mounted ||= @backend.run_command("lsfs -c #{@spath}")
+        end
+      end
+    end
+  end
+end

data/lib/train/file/remote/linux.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+require 'train/file/remote/unix'
+
+module Train
+  class File
+    class Remote
+      class Linux < Train::File::Remote::Unix
+        def content
+          return @content if defined?(@content)
+          @content = @backend.run_command("cat #{@path} || echo -n").stdout
+          return @content unless @content.empty?
+          @content = nil if directory? or size.nil? or size > 0
+          @content
+        end
+      end
+    end
+  end
+end

data/lib/train/file/remote/qnx.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+#
+# author: Christoph Hartmann
+# author: Dominik Richter
+
+require 'train/file/remote/unix'
+
+module Train
+  class File
+    class Remote
+      class Aix < Train::File::Remote::Unix
+        def content
+          cat = 'cat'
+          cat = '/proc/boot/cat' if @backend.os[:release].to_i >= 7
+          @content ||= case
+                       when !exist?
+                         nil
+                       else
+                         @backend.run_command("#{cat} #{@spath}").stdout || ''
+                       end
+        end
+
+        def type
+          if @backend.run_command("file #{@spath}").stdout.include?('directory')
+            return :directory
+          else
+            return :file
+          end
+        end
+
+        %w{
+          mode owner group uid gid mtime size selinux_label link_path mounted stat
+        }.each do |field|
+          define_method field.to_sym do
+            fail NotImplementedError, "QNX does not implement the #{m}() method yet."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/train/file/remote/unix.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+
+require 'shellwords'
+
+module Train
+  class File
+    class Remote
+      class Unix < Train::File::Remote
+        attr_reader :path
+
+        def sanitize_filename(path)
+          @spath = Shellwords.escape(path) || @path
+        end
+
+        def content
+          @content ||=
+            if !exist? || directory?
+              nil
+            elsif size.nil? || size.zero?
+              ''
+            else
+              @backend.run_command("cat #{@spath}").stdout || ''
+            end
+        end
+
+        def exist?
+          @exist ||= (
+            f = @follow_symlink ? '' : " || test -L #{@spath}"
+            @backend.run_command("test -e #{@spath}"+f)
+                    .exit_status == 0
+          )
+        end
+
+        def mounted?
+          !mounted.nil? && !mounted.stdout.nil? && !mounted.stdout.empty?
+        end
+
+        def mounted
+          @mounted ||=
+            @backend.run_command("mount | grep -- ' on #{@spath} '")
+        end
+
+        %w{
+          type mode owner group uid gid mtime size selinux_label
+        }.each do |field|
+          define_method field.to_sym do
+            stat[field.to_sym]
+          end
+        end
+
+        def mode?(sth)
+          mode == sth
+        end
+
+        def grouped_into?(sth)
+          group == sth
+        end
+
+        def linked_to?(dst)
+          link_path == dst
+        end
+
+        def link_path
+          symlink? ? path : nil
+        end
+
+        def unix_mode_mask(owner, type)
+          o = UNIX_MODE_OWNERS[owner.to_sym]
+          return nil if o.nil?
+
+          t = UNIX_MODE_TYPES[type.to_sym]
+          return nil if t.nil?
+
+          t & o
+        end
+
+        def path
+          return @path unless @follow_symlink && symlink?
+          @link_path ||= read_target_path
+        end
+
+        private
+
+        # Returns full path of a symlink target(real dest) or '' on symlink loop
+        def read_target_path
+          full_path = @backend.run_command("readlink -n #{@spath} -f").stdout
+          # Needed for some OSes like OSX that returns relative path
+          # when the link and target are in the same directory
+          if !full_path.start_with?('/') && full_path != ''
+            full_path = ::File.expand_path("../#{full_path}", @spath)
+          end
+          full_path
+        end
+
+        UNIX_MODE_OWNERS = {
+          all:   00777,
+          owner: 00700,
+          group: 00070,
+          other: 00007,
+        }.freeze
+
+        UNIX_MODE_TYPES = {
+          r: 00444,
+          w: 00222,
+          x: 00111,
+        }.freeze
+      end
+    end
+  end
+end

data/lib/train/file/remote/windows.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+module Train
+  class File
+    class Remote
+      class Windows < Train::File::Remote
+        attr_reader :path
+        # Ensures we do not use invalid characters for file names
+        # @see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#naming_conventions
+        def sanitize_filename(path)
+          return if path.nil?
+          # we do not filter :, backslash and forward slash, since they are part of the path
+          @spath = path.gsub(/[<>"|?*]/, '')
+        end
+
+        def basename(suffix = nil, sep = '\\')
+          super(suffix, sep)
+        end
+
+        def content
+          return @content if defined?(@content)
+          @content = @backend.run_command("Get-Content(\"#{@spath}\") | Out-String").stdout
+          return @content unless @content.empty?
+          @content = nil if directory? # or size.nil? or size > 0
+          @content
+        end
+
+        def exist?
+          return @exist if defined?(@exist)
+          @exist = @backend.run_command(
+            "(Test-Path -Path \"#{@spath}\").ToString()").stdout.chomp == 'True'
+        end
+
+        def owner
+          owner = @backend.run_command(
+            "Get-Acl \"#{@spath}\" | select -expand Owner").stdout.strip
+          return if owner.empty?
+          owner
+        end
+
+        def type
+          if attributes.include?('Archive')
+            return :file
+          elsif attributes.include?('ReparsePoint')
+            return :symlink
+          elsif attributes.include?('Directory')
+            return :directory
+          end
+          :unknown
+        end
+
+        def size
+          if file?
+            @backend.run_command("((Get-Item '#{@spath}').Length)").stdout.strip.to_i
+          end
+        end
+
+        def product_version
+          @product_version ||= @backend.run_command(
+            "[System.Diagnostics.FileVersionInfo]::GetVersionInfo(\"#{@spath}\").ProductVersion").stdout.chomp
+        end
+
+        def file_version
+          @file_version ||= @backend.run_command(
+            "[System.Diagnostics.FileVersionInfo]::GetVersionInfo(\"#{@spath}\").FileVersion").stdout.chomp
+        end
+
+        %w{
+          mode group uid gid mtime selinux_label
+        }.each do |field|
+          define_method field.to_sym do
+            nil
+          end
+        end
+
+        def link_path
+          nil
+        end
+
+        def mounted
+          nil
+        end
+
+        private
+
+        def attributes
+          return @attributes if defined?(@attributes)
+          @attributes = @backend.run_command(
+            "(Get-ItemProperty -Path \"#{@spath}\").attributes.ToString()").stdout.chomp.split(/\s*,\s*/)
+        end
+      end
+    end
+  end
+end

data/lib/train/plugins/base_connection.rb

@@ -6,6 +6,7 @@
 
 require 'train/errors'
 require 'train/extras'
+require 'train/file'
 require 'logger'
 
 class Train::Plugins::Transport
@@ -78,7 +79,7 @@ class Train::Plugins::Transport
     #
     # @return [LoginCommand] array of command line tokens
     def login_command
-      fail Train::ClientError, "#{self.class} does not implement #run_command()"
+      fail Train::ClientError, "#{self.class} does not implement #login_command()"
     end
 
     # Block and return only when the remote host is prepared and ready to

data/lib/train/transports/docker.rb

@@ -74,7 +74,14 @@ class Train::Transports::Docker
     end
 
     def file(path)
-      @files[path] ||= LinuxFile.new(self, path)
+      @files[path] ||=\
+        if os.aix?
+          Train::File::Remote::Aix.new(self, path)
+        elsif os.solaris?
+          Train::File::Remote::Unix.new(self, path)
+        else
+          Train::File::Remote::Linux.new(self, path)
+        end
     end
 
     def run_command(cmd)

data/lib/train/transports/local.rb

@@ -17,7 +17,6 @@ module Train::Transports
     end
 
     class Connection < BaseConnection
-      require 'train/transports/local_file'
       require 'train/transports/local_os'
 
       def initialize(options)
@@ -40,7 +39,12 @@ module Train::Transports
       end
 
       def file(path)
-        @files[path] ||= File.new(self, path)
+        @files[path] ||= \
+          if os.windows?
+            Train::File::Local::Windows.new(self, path)
+          else
+            Train::File::Local::Unix.new(self, path)
+          end
       end
 
       def login_command

data/lib/train/transports/mock.rb

@@ -65,7 +65,7 @@ class Train::Transports::Mock
 
     def initialize(conf = nil)
       super(conf)
-      @os = OS.new(self, family: 'unknown')
+      mock_os
       @commands = {}
     end
 
@@ -73,8 +73,9 @@ class Train::Transports::Mock
       'mock://'
     end
 
-    def mock_os(value)
-      @os = OS.new(self, value)
+    def mock_os(value = {})
+      os_params = { name: 'unknown', family: 'unknown', release: 'unknown', arch: 'unknown' }.merge(value)
+      @os = OS.new(self, os_params)
     end
 
     def mock_command(cmd, stdout = nil, stderr = nil, exit_status = 0)
@@ -129,20 +130,20 @@ class Train::Transports::Mock::Connection
 end
 
 class Train::Transports::Mock::Connection
-  class File < FileCommon
+  class File < Train::File
     def self.from_json(json)
       res = new(json['backend'],
                 json['path'],
                 json['follow_symlink'])
       res.type = json['type']
-      Train::Extras::FileCommon::DATA_FIELDS.each do |f|
+      Train::File::DATA_FIELDS.each do |f|
         m = (f.tr('?', '') + '=').to_sym
         res.method(m).call(json[f])
       end
       res
     end
 
-    Train::Extras::FileCommon::DATA_FIELDS.each do |m|
+    Train::File::DATA_FIELDS.each do |m|
       attr_accessor m.tr('?', '').to_sym
       next unless m.include?('?')
 

data/lib/train/transports/ssh.rb

@@ -97,8 +97,7 @@ module Train::Transports
       if options[:auth_methods] == ['none']
         if ssh_known_identities.empty?
           fail Train::ClientError,
-               'You must configure at least one authentication method for SSH:'\
-               ' Agent, Key or Password.'
+               'Your SSH Agent has no keys added, and you have not specified a password or a key file'
         else
           logger.debug('[SSH] Using Agent keys as no password or key file have been specified')
           options[:auth_methods].push('publickey')

data/lib/train/transports/ssh_connection.rb

@@ -39,6 +39,7 @@ class Train::Transports::SSH
       @connection_retries     = @options.delete(:connection_retries)
       @connection_retry_sleep = @options.delete(:connection_retry_sleep)
       @max_wait_until_ready   = @options.delete(:max_wait_until_ready)
+      @max_ssh_sessions       = @options.delete(:max_ssh_connections) { 9 }
       @session                = nil
       @transport_options      = @options.delete(:transport_options)
       @cmd_wrapper            = nil
@@ -61,11 +62,13 @@ class Train::Transports::SSH
     def file(path)
       @files[path] ||= \
         if os.aix?
-          AixFile.new(self, path)
+          Train::File::Remote::Aix.new(self, path)
         elsif os.solaris?
-          UnixFile.new(self, path)
+          Train::File::Remote::Unix.new(self, path)
+        elsif os[:name] == 'qnx'
+          Train::File::Remote::Qnx.new(self, path)
         else
-          LinuxFile.new(self, path)
+          Train::File::Remote::Linux.new(self, path)
         end
     end
 
@@ -134,17 +137,34 @@ class Train::Transports::SSH
 
     # (see Base::Connection#upload)
     def upload(locals, remote)
+      waits = []
       Array(locals).each do |local|
         opts = File.directory?(local) ? { recursive: true } : {}
 
-        session.scp.upload!(local, remote, opts) do |_ch, name, sent, total|
+        waits.push session.scp.upload(local, remote, opts) do |_ch, name, sent, total|
           logger.debug("Uploaded #{name} (#{total} bytes)") if sent == total
         end
+        waits.shift.wait while waits.length >= @max_ssh_sessions
       end
+      waits.each(&:wait)
     rescue Net::SSH::Exception => ex
       raise Train::Transports::SSHFailed, "SCP upload failed (#{ex.message})"
     end
 
+    def download(remotes, local)
+      waits = []
+      Array(remotes).map do |remote|
+        opts = file(remote).directory? ? { recursive: true } : {}
+        waits.push session.scp.download(remote, local, opts) do |_ch, name, recv, total|
+          logger.debug("Downloaded #{name} (#{total} bytes)") if recv == total
+        end
+        waits.shift.wait while waits.length >= @max_ssh_sessions
+      end
+      waits.each(&:wait)
+    rescue Net::SSH::Exception => ex
+      raise Train::Transports::SSHFailed, "SCP download failed (#{ex.message})"
+    end
+
     # (see Base::Connection#wait_until_ready)
     def wait_until_ready
       delay = 3