rest-ftp-daemon 0.423.3 → 0.424.0

data/lib/rest-ftp-daemon/remote/base.rb

@@ -0,0 +1,76 @@
+# Handles transfers for Job class
+module RestFtpDaemon
+  module Remote
+    class RemoteBase
+      include BmcDaemonLib::LoggerHelper
+
+      # Class options
+      attr_reader :logger
+      attr_reader :log_prefix
+      attr_accessor :job
+
+      # Delegate set_info info to Job
+      delegate :set_info, to: :job
+
+      def initialize target, log_prefix, debug = false, ftpes = false
+        # Init
+        @target = target
+        @ftpes = ftpes
+        @debug = !!debug
+
+        # Build and empty job to protect set_info delegation
+        @job = Job.new(nil, {})
+
+        # Logger
+        @log_prefix = log_prefix || {}
+        @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
+
+        # Annnounce object
+        log_info "Remote.initialize debug[#{debug}] target[#{target.path}] "
+
+        # Prepare real object
+        prepare
+      end
+
+      def prepare
+      end
+
+      def connect
+        # Debug mode ?
+        return unless @debug
+        puts
+        puts "-------------------- SESSION STARTING -------------------------"
+        puts "class\t #{myname}"
+        puts "host\t #{@target.host}"
+        puts "user\t #{@target.user}"
+        puts "port\t #{@target.port}"
+        puts "---------------------------------------------------------------"
+      end
+
+      def chdir_or_create directory, mkdir = false
+      end
+
+      def remove! target
+      end
+
+      def close
+        # Debug mode ?
+        return unless @debug
+        puts "-------------------- SESSION CLOSING --------------------------"
+      end
+
+    private
+
+      def extract_parent path
+        return unless path.is_a? String
+        m = path.match(/^(.*)\/([^\/]+)\/?$/)
+        return m[1], m[2] unless m.nil?
+      end
+
+      def myname
+        self.class.to_s
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/remote/ftp.rb

@@ -0,0 +1,144 @@
+require "net/ftp"
+require "double_bag_ftps"
+
+# Handle FTP and FTPeS transfers for Remote class
+module RestFtpDaemon
+  module Remote
+    class RemoteFTP < RemoteBase
+
+      # Class options
+      attr_reader :ftp
+
+      def prepare
+        # Create FTP object
+        if @ftpes
+          prepare_ftpes
+        else
+          prepare_ftp
+        end
+        @ftp.passive = true
+        @ftp.debug_mode = @debug
+
+        # Config
+        @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
+
+        # Announce object
+        log_debug "Remote::RemoteFTP.prepare chunk_size:#{@chunk_size}"
+      end
+
+      def connect
+        super
+
+        # Connect remote server
+        @ftp.connect @target.host, @target.port
+        @ftp.login @target.user, @target.password
+      end
+
+      def present? target
+        size = @ftp.size target.path
+        log_debug "Remote::RemoteFTP.present? [#{target.name}]"
+
+      rescue Net::FTPPermError
+        return false
+      else
+        return size
+      end
+
+      def remove! target
+        @ftp.delete target.path
+      rescue Net::FTPPermError
+        log_debug "Remote::RemoteFTP.remove! [#{target.name}] not found"
+      else
+        log_debug "Remote::RemoteFTP.remove! [#{target.name}] removed"
+      end
+
+      def mkdir directory
+        log_debug "Remote::RemoteFTP.mkdir [#{directory}]"
+        @ftp.mkdir directory
+
+      rescue StandardError => ex
+        raise TargetPermissionError, ex.message
+      end
+
+      def chdir_or_create directory, mkdir = false
+        # Init, extract my parent name and my own name
+        log_debug "Remote::RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+        parent, current = extract_parent(directory)
+
+        #dirname, _current = extract_parent(directory)
+
+
+        # Access this directory
+        begin
+          @ftp.chdir "/#{directory}"
+
+        rescue Net::FTPPermError => _e
+          # If not allowed to create path, that's over, we're stuck
+          return false unless mkdir
+          chdir_or_create parent, mkdir
+
+          # Now I was able to chdir into my parent, create the current directory
+          mkdir current
+
+          # Finally retry the chdir
+          retry
+        else
+          return true
+        end
+      end
+
+      def upload source, target, use_temp_name = false, &callback
+        # Push init
+        raise RestFtpDaemon::AssertionFailed, "upload/ftp" if @ftp.nil?
+
+        # Temp file if needed
+        dest = target.clone
+        if use_temp_name
+          dest.generate_temp_name!
+        end
+
+        # Move to the directory
+        log_debug "Remote::RemoteFTP.upload chdir [#{dest.dir}]"
+        @ftp.chdir "/#{dest.dir}"
+
+        # Do the transfer
+        log_debug "Remote::RemoteFTP.upload putbinaryfile [#{dest.name}]"
+        @ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
+          # Update job status after this block transfer
+          yield data.bytesize, dest.name
+        end
+
+        # Move the file back to its original name
+        if use_temp_name
+          log_debug "Remote::RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
+          @ftp.rename dest.name, target.name
+        end
+      end
+
+      def close
+        # Close init
+        super
+
+        # Close FTP connexion and free up memory
+        @ftp.close
+      end
+
+      def connected?
+        !@ftp.welcome.nil?
+      end
+
+    private
+
+      def prepare_ftp
+        @ftp = Net::FTP.new
+      end
+
+      def prepare_ftpes
+        @ftp = DoubleBagFTPS.new
+        @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
+        @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/remote/s3.rb

@@ -0,0 +1,78 @@
+require 'aws-sdk-resources'
+
+# Handle sFTP transfers for Remote class
+module RestFtpDaemon
+  module Remote
+    class RemoteS3 < RemoteBase
+
+      MULTIPART_THRESHOLD_MB = 4
+
+      # Class options
+      attr_reader :client
+      attr_reader :target
+
+      def prepare
+        @multipart_threshold = MULTIPART_THRESHOLD_MB.to_i * 1024 * 1024
+        log_debug "Remote::RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
+      end
+
+      def connect
+        super
+
+        # Connect init
+        log_debug "Remote::RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
+
+        # Connect remote server
+        @client = Aws::S3::Resource.new(
+          region: @target.aws_region,
+          credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
+          http_wire_trace: @debug
+          )
+        #s3 = Aws::S3::Client.new(http_wire_trace: true)
+      end
+
+      def upload source, target, use_temp_name = false, &callback
+        # Push init
+        raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
+        log_debug "Remote::RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
+
+        # Update progress before
+        #yield 0, target.name
+        # Point to the right bucket and object
+        bucket = @client.bucket(target.aws_bucket)
+        object = bucket.object(target.name)
+
+        # Do the transfer
+        object.upload_file(source.path, {
+          multipart_threshold: @multipart_threshold
+          })
+
+        # Wait for transfer to complete
+        object.wait_until_exists do |waiter|
+          # waiter.delay = 1
+          # # log_debug "- progress[#{progress}] total[#{total}]"
+          # waiter.before_wait do |attempts, response|
+          #   puts "#{attempts} made"
+          #   puts response.error.inspect
+          #   puts response.data.inspect
+          # end
+          # log_debug "- progress[] #{waiter.inspect}"
+        end
+
+        # Update progress after
+        #yield target.size, target.name
+
+        # Dump information about this file
+        log_debug "Remote::RemoteS3.upload url[#{object.public_url}]"
+        log_debug "Remote::RemoteS3.upload etag[#{object.etag}]"
+        set_info :target_aws_public_url, object.public_url
+        set_info :target_aws_etag, object.etag
+      end
+
+      def connected?
+        !@client.nil?
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/remote/sftp.rb

@@ -0,0 +1,147 @@
+require "net/sftp"
+
+# Handle sFTP transfers for Remote class
+module RestFtpDaemon
+  module Remote
+    class RemoteSFTP < RemoteBase
+
+      # Class options
+      attr_reader :sftp
+
+      def prepare
+      end
+
+      def connect
+        super
+
+        # Connect init
+        log_debug "Remote::RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
+
+        # Debug level
+        verbosity =  @debug ? Logger::DEBUG : false
+
+        # Connect remote server
+        @sftp = Net::SFTP.start(@target.host.to_s, @target.user.to_s,
+            password: @target.password.to_s,
+            verbose: verbosity,
+            port: @target.port,
+            non_interactive: true,
+            timeout: DEFAULT_SFTP_TIMEOUT
+            )
+      end
+
+      def present? target
+        log_debug "Remote::RemoteSFTP.present? [#{target.name}]"
+        stat = @sftp.stat! target.path
+
+      rescue Net::SFTP::StatusException
+        return false
+      else
+        return stat.size
+      end
+
+      def remove! target
+        log_debug "Remote::RemoteSFTP.remove! [#{target.name}]"
+        @sftp.remove target.path
+
+      rescue Net::SFTP::StatusException
+        log_debug "#{LOG_INDENT}[#{target.name}] file not found"
+      else
+        log_debug "#{LOG_INDENT}[#{target.name}] removed"
+      end
+
+      def mkdir directory
+        log_debug "Remote::RemoteSFTP.mkdir [#{directory}]"
+        @sftp.mkdir! directory
+
+        rescue StandardError => ex
+          raise TargetPermissionError, ex.message
+      end
+
+      def chdir_or_create directory, mkdir = false
+        # Init, extract my parent name and my own name
+        log_debug "Remote::RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+        parent, _current = extract_parent(directory)
+
+        # Access this directory
+        begin
+          #log_debug "chdir [/#{directory}]"
+          @sftp.opendir! directory
+
+        rescue Net::SFTP::StatusException => _e
+          # If not allowed to create path, that's over, we're stuck
+          return false unless mkdir
+
+          # Recurse upward
+          chdir_or_create parent, mkdir
+
+          # Now I was able to chdir into my parent, create the current directory
+          mkdir directory
+
+          # Finally retry the chdir
+          retry
+        else
+          return true
+        end
+
+        # We should never get here
+        raise JobTargetShouldBeDirectory
+      end
+
+      def upload source, target, use_temp_name = false, &callback
+        # Push init
+        raise RestFtpDaemon::AssertionFailed, "upload/sftp" if @sftp.nil?
+
+        # Temp file if needed
+        dest = target.clone
+        if use_temp_name
+          dest.generate_temp_name!
+        end
+
+        # Do the transfer
+        log_debug "Remote::RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
+        @sftp.upload! source.path, dest.path do |event, _uploader, *args|
+          case event
+          when :open then
+            # args[0] : file metadata
+          when :put then
+            # args[0] : file metadata
+            # args[1] : byte offset in remote file
+            # args[2] : data being written (as string)
+            # puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
+
+            # Update job status after this block transfer
+            yield args[2].length, dest.name
+
+          when :close then
+            # args[0] : file metadata
+          when :mkdir
+            # args[0] : remote path name
+          when :finish
+          end
+
+        end
+
+        # Move the file back to its original name
+        if use_temp_name
+          flags = 0x00000001
+          log_debug "Remote::RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
+          @sftp.rename! dest.path, target.path, flags
+        end
+
+        # progress:
+        # Net::SFTP::StatusException
+      end
+
+      def close
+        # Close init
+        super
+      end
+
+      def connected?
+        @sftp && !@sftp.closed?
+      end
+
+    end
+  end
+end