rest-ftp-daemon 0.410.5 → 0.420.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0fa821b4974f3c450b6ba939b7d550056f567f61
-  data.tar.gz: 5e63689d2ef91e6a28046824086da73e1519f78f
+  metadata.gz: a1dc08f8d5c0fc9a671b36144561e504e25bb80e
+  data.tar.gz: 9e8eae9daf0a21d56f6fe6c27a32d19327fc099e
 SHA512:
-  metadata.gz: 38976cb5fbdd2ba3adafc8423eec17b2db16c9b022ad53b7c34ce3b57cf2f936a26b328e7abdbb6483a337597d852b35e3f94a5aef5c1587391856db6eda7ba8
-  data.tar.gz: 63cf9c1429b5ac2ffcbb7a0a3a7160de6e1ebedbc74505b399e4d437ee30fe323cd9533c7cb1933aff649a4fbb9576d0652da710df511a74a9a8f0cbb377f653
+  metadata.gz: 08d906444c38c286123f1eacef7f7c37ba30ff64ea9a4da83913c745514288eeefda8ee8bdcfe1f8de52f2c29132cb2fcfa902f978b600fc387439970978b6e8
+  data.tar.gz: d4a131e6f37dca12e5478443e2bfffb2975de3ae59481c67c47428eedabaeee045866a17640c0e1875be5357cdb29726f3052f4b314fe0660bc4d6dd15a0db66

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.410.5)
+    rest-ftp-daemon (0.420.1)
       activesupport (~> 4.2)
       api-auth
+      aws-sdk-resources (~> 2)
       bmc-daemon-lib (~> 0.3.4)
       double-bag-ftps
       facter
@@ -38,6 +39,10 @@ GEM
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
+    aws-sdk-core (2.5.7)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.5.7)
+      aws-sdk-core (= 2.5.7)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
@@ -102,6 +107,7 @@ GEM
     http_parser.rb (0.6.0)
     i18n (0.7.0)
     ice_nine (0.11.2)
+    jmespath (1.3.1)
     json (1.8.3)
     method_source (0.8.2)
     mime-types (2.99.2)

data/bin/rest-ftp-daemon

@@ -59,11 +59,11 @@ begin
   command = ARGV.shift
 
 rescue OptionParser::InvalidOption => e
-  abort "EXITING: InvalidOption: #{e.message} \n #{e.backtrace.to_yaml}"
+  abort "EXITING: InvalidOption: #{e.message}"
 rescue ConfigParseError => e
-  abort "EXITING: ConfigParseError: #{e.message} \n #{e.backtrace.to_yaml}"
+  abort "EXITING: ConfigParseError: #{e.message}"
 rescue StandardError => e
-  abort "EXITING: StandardError: #{e.message} \n #{e.backtrace.to_yaml}"
+  abort "EXITING: StandardError: #{e.message}"
 else
   abort parser.to_s unless ["start", "stop"].include? command
 end

data/defaults.yml

@@ -29,6 +29,7 @@ transfer:
   timeout: 1800             # jobs running for longer than X seconds will be killed
   notify_after: 5           # wait at least X seconds between HTTP notifications
 
+  debug_s3: false
   debug_ftp: false
   debug_ftps: false
   debug_sftp: false

data/lib/rest-ftp-daemon.rb

@@ -26,11 +26,13 @@ require_relative "rest-ftp-daemon/job_queue"
 require_relative "rest-ftp-daemon/counters"
 require_relative "rest-ftp-daemon/notification"
 require_relative "rest-ftp-daemon/location"
+require_relative "rest-ftp-daemon/pretty_json"
 
 # Remotes
 require_relative "rest-ftp-daemon/remote"
 require_relative "rest-ftp-daemon/remote_ftp"
 require_relative "rest-ftp-daemon/remote_sftp"
+require_relative "rest-ftp-daemon/remote_s3"
 
 # Jobs
 require_relative "rest-ftp-daemon/job"

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -7,6 +7,7 @@ require "grape"
 module RestFtpDaemon
   module API
     class Dashbaord < Grape::API
+      include BmcDaemonLib
 
       ### HELPERS
       helpers do

data/lib/rest-ftp-daemon/api/debug.rb

@@ -3,6 +3,7 @@ require "grape"
 module RestFtpDaemon
   module API
     class Debug < Grape::API
+      include BmcDaemonLib
 
       ### HELPERS
       helpers do

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -3,6 +3,7 @@ require "grape"
 module RestFtpDaemon
   module API
     class Jobs < Grape::API
+      include BmcDaemonLib
 
       ### ENDPOINTS
       desc "Read job with ID", http_codes: [

data/lib/rest-ftp-daemon/api/root.rb

@@ -7,6 +7,7 @@ module RestFtpDaemon
   module API
     class Root < Grape::API
       include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      include BmcDaemonLib
 
       ### LOGGING & HELPERS
       helpers RestFtpDaemon::CommonHelpers
@@ -33,6 +34,7 @@ module RestFtpDaemon
       do_not_route_options!
       # version 'v1'
       format :json
+      formatter :json, PrettyJSON
       content_type :json, 'application/json; charset=utf-8'
 
 

data/lib/rest-ftp-daemon/api/status.rb

@@ -4,6 +4,7 @@ require "get_process_mem"
 module RestFtpDaemon
   module API
     class Status < Grape::API
+      include BmcDaemonLib
 
       ### ENDPOINTS
       desc "Show daemon status"

data/lib/rest-ftp-daemon/helpers/views.rb

@@ -38,6 +38,8 @@ module RestFtpDaemon
         "warning"
       when URI::FTPES, URI::FTPS, URI::SFTP
         "success"
+      when URI::S3
+        "primary"
       else
         "default"
       end

data/lib/rest-ftp-daemon/job.rb

@@ -43,9 +43,13 @@ module RestFtpDaemon
       attr_reader name
     end
 
-    def initialize job_id, params = {}
-      # Call super
-      # super()
+    def initialize job_id = nil, params = {}
+      # Minimal init
+      @infos = {}
+      @mutex = Mutex.new
+
+      # Skip if no job_id passed or null (mock Job)
+      return if job_id.nil?
 
       # Init context
       @id = job_id.to_s
@@ -56,7 +60,6 @@ module RestFtpDaemon
       @status = nil
       @runs = 0
       @wid = nil
-      @infos = {}
 
       # Prepare configuration
       @config       = Conf[:transfer] || {}
@@ -66,8 +69,6 @@ module RestFtpDaemon
       # Logger
       @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
 
-      # Protect with a mutex
-      @mutex = Mutex.new
 
       # Import query params
       FIELDS.each do |name|
@@ -101,7 +102,6 @@ module RestFtpDaemon
       # Job has been prepared, reset infos
       set_status JOB_STATUS_PREPARED
       @infos = {}
-      set_info :job, :prepared_at, Time.now
       set_info_location :source, @source_loc
       set_info_location :target, @target_loc
 
@@ -140,7 +140,7 @@ module RestFtpDaemon
       do_after
 
     rescue StandardError => exception
-      log_debug "Job.process caught #{exception.class} #{exception.message}"
+      log_debug "Job.process caught [#{exception.class}] #{exception.message}"
       return oops current_signal, exception
 
     else
@@ -214,6 +214,23 @@ module RestFtpDaemon
       end
     end
 
+    def set_info level1, level2, value
+      @mutex.synchronize do
+        @infos || {}
+        @infos[level1] ||= {}
+
+        # Force strings to UTF8
+        if value.is_a? Symbol
+          @infos[level1][level2] = value.to_s.force_encoding(Encoding::UTF_8)
+        elsif value.is_a? String
+          @infos[level1][level2] = value.dup.force_encoding(Encoding::UTF_8)
+        else
+          @infos[level1][level2] = value
+        end
+      end
+      touch_job
+    end
+
   protected
 
     def alert_common_method_called
@@ -236,8 +253,13 @@ module RestFtpDaemon
       return unless location.is_a? Location
       set_info prefix, :uri,    location.to_s
       set_info prefix, :scheme, location.scheme
+      set_info prefix, :user,   location.user
       set_info prefix, :host,   location.host
+      set_info prefix, :port,   location.port
       set_info prefix, :path,   location.path
+      set_info prefix, :aws_region,   location.aws_region
+      set_info prefix, :aws_bucket,   location.aws_bucket
+      set_info prefix, :aws_id,       location.aws_id
     end
 
   private
@@ -266,23 +288,6 @@ module RestFtpDaemon
       touch_job
     end
 
-    def set_info level1, level2, value
-      @mutex.synchronize do
-        @infos || {}
-        @infos[level1] ||= {}
-
-        # Force strings to UTF8
-        if value.is_a? Symbol
-          @infos[level1][level2] = value.to_s.force_encoding(Encoding::UTF_8)
-        elsif value.is_a? String
-          @infos[level1][level2] = value.dup.force_encoding(Encoding::UTF_8)
-        else
-          @infos[level1][level2] = value
-        end
-      end
-      touch_job
-    end
-
     def utf8 value
       value.to_s.encode("UTF-8")
     end

data/lib/rest-ftp-daemon/jobs/errors.rb

@@ -17,6 +17,7 @@ module RestFtpDaemon
       target_file_exists:       RestFtpDaemon::TargetFileExists,
       target_directory_error:   RestFtpDaemon::TargetDirectoryError,
       target_permission_error:  RestFtpDaemon::TargetPermissionError,
+      target_not_supported:     RestFtpDaemon::TargetNotSupported,
       assertion_failed:         RestFtpDaemon::AssertionFailed,
 
       conn_socket_error:        SocketError,

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -18,23 +18,28 @@ module RestFtpDaemon
       # Ensure source is FILE
       raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme   unless source_uri.is_a? URI::FILE
 
-      # Prepare remote (case would be preferable but too hard to use,
-      # as target could be of a descendent class of URI:XXX and not matching directly)
+      # Prepare remote object
       case target_uri
       when URI::FTP
         log_info "JobTransfer.before target_method FTP"
-        @remote = RemoteFTP.new @target_loc.uri, log_prefix, debug: @config[:debug_ftp]
+        @remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps]
       when URI::FTPES, URI::FTPS
         log_info "JobTransfer.before target_method FTPES/FTPS"
-        @remote = RemoteFTP.new @target_loc.uri, log_prefix, debug: @config[:debug_ftps], ftpes: true
+        @remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
       when URI::SFTP
         log_info "JobTransfer.before target_method SFTP"
-        @remote = RemoteSFTP.new @target_loc.uri, log_prefix, debug: @config[:debug_sftp]
+        @remote = RemoteSFTP.new @target_loc, log_prefix, @config[:debug_sftp]
+      when URI::S3
+        log_info "JobTransfer.before target_method S3"
+        @remote = RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
       else
         log_info "JobTransfer.before unknown scheme [#{@target_loc.scheme}]"
         raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme
       end
 
+      # Plug this Job into @remote to allow it to log
+      @remote.job = self
+
     # rescue URI::InvalidURIError => exception
     #   return oops :started, exception, "target_invalid"
     end
@@ -84,7 +89,7 @@ module RestFtpDaemon
         remote_push source, target_final
 
         # Add it to transferred target names
-        targets << target_final.path
+        targets << target_final.name
         set_info :target, :files, targets
 
         # Update counters

data/lib/rest-ftp-daemon/location.rb

@@ -4,21 +4,27 @@ require 'active_support/core_ext/module/delegation'
 module RestFtpDaemon
   class Location
     # Accessors
+    attr_accessor :name
+
     attr_reader :uri
     attr_reader :scheme
     attr_reader :dir
-    attr_accessor :name
+
+
+    attr_reader :aws_region
+    attr_reader :aws_bucket
+    attr_reader :aws_id
+    attr_reader :aws_secret
 
     # Logging
     #attr_reader :logger
     #include BmcDaemonLib::LoggerHelper
 
     # def_delegators :@uri,
-    delegate :scheme, :host, :user, :password, :to_s, to: :uri
+    delegate :scheme, :host, :port, :user, :password, :to_s,
+      to: :uri
 
     def initialize path
-      #@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
-      #log_debug "Location.initialize path[#{path}]"
       # Strip spaces before/after, copying original "path" at the same time
       location_uri = path.strip
 
@@ -27,12 +33,16 @@ module RestFtpDaemon
       fix_scheme! location_uri
 
       # Parse URL
-      #log_debug "Location.initialize uri[#{location_uri}]"
       parse_url location_uri
 
+      # Match AWS URL with BUCKET.s3.amazonaws.com
+      init_aws if @uri.is_a? URI::S3
+
+      # Set default user if not provided
+      init_username
+
       # Ensure result does not contain tokens after replacement
       detected_tokens = detect_tokens(location_uri)
-      #log_debug "Location.initialize detected_tokens: #{detected_tokens.inspect}"
       unless detected_tokens.empty?
         raise RestFtpDaemon::UnresolvedTokens, detected_tokens.join(' ')
       end
@@ -41,13 +51,6 @@ module RestFtpDaemon
       unless @uri.scheme
         raise RestFtpDaemon::UnsupportedScheme, url
       end
-
-      # All done
-      #log_debug "Location.initialize class[#{@uri.class}] scheme[#{@uri.scheme}] dir[#{@dir}] name[#{@name}]"
-    end
-
-    def is? klass
-      @uri.is_a? klass
     end
 
     def path
@@ -88,7 +91,7 @@ module RestFtpDaemon
 
       # Replace endpoints defined in config
       vectors.each do |from, to|
-        next if to.to_s.blank?
+        next if to.to_s.empty?
         path.gsub! tokenize(from), to
       end
     end
@@ -105,10 +108,9 @@ module RestFtpDaemon
     def parse_url path
       # Parse that URL
       @uri = URI.parse path # rescue nil
-      raise RestFtpDaemon::LocationParseError, location_path unless uri
+      raise RestFtpDaemon::LocationParseError, location_path unless @uri
 
       # Store URL parts
-      # remove_multiple_slashes
       @ori_path = path
       @uri_path = uri.path
       @dir      = extract_dirname uri.path
@@ -118,6 +120,26 @@ module RestFtpDaemon
         raise RestFtpDaemon::LocationParseError, exception.message unless uri
     end
 
+    def init_username
+      @uri.user ||= "anonymous"
+    end
+
+    def init_aws
+      # Split hostname
+      parts       = @uri.host.split('.')
+
+      # Pop parts
+      aws_tld     = parts.pop
+      aws_domain  = parts.pop
+      @aws_region = parts.pop
+      aws_tag     = parts.pop
+      @aws_bucket = parts.pop
+
+      # Credentials from config
+      @aws_id     = Conf.at(:credentials, @uri.host, :id)
+      @aws_secret = Conf.at(:credentials, @uri.host, :secret)
+    end
+
     def extract_filename path
       # match everything that's after a slash at the end of the string
       m = path.match(/\/?([^\/]+)$/)

data/lib/rest-ftp-daemon/pretty_json.rb

@@ -0,0 +1,5 @@
+module PrettyJSON
+  def self.call(object, env)
+    JSON.pretty_generate(JSON.parse(object.to_json))
+  end
+end

data/lib/rest-ftp-daemon/remote.rb

@@ -7,20 +7,33 @@ module RestFtpDaemon
     attr_reader :logger
     attr_reader :log_prefix
 
-    def initialize url, log_prefix, options = {}
-      # Options
-      @debug = !!options[:debug]
+    attr_accessor :job
+
+    # Delegate set_info info to Job
+    delegate :set_info, to: :job
+
+    def initialize target, log_prefix, debug = false, ftpes = false
+      # Init
+      @target = target
+      @ftpes = ftpes
+      @debug = debug
+
+      # Build and empty job to protect set_info delegation
+      @job = Job.new(nil, {})
 
       # Logger
       @log_prefix = log_prefix || {}
       @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
 
-      # Extract URL parts
-      @url = url
-      @url.user ||= "anonymous"
-
       # Annnounce object
-      log_info "Remote.initialize [#{url}]"
+      log_info "Remote.initialize [#{target.path}]"
+      log_debug "Remote.initialize target[#{@target.inspect}]"
+
+      # Prepare real object
+      prepare
+    end
+
+    def prepare
     end
 
     def connect
@@ -29,12 +42,16 @@ module RestFtpDaemon
       puts
       puts "-------------------- SESSION STARTING -------------------------"
       puts "class\t #{myname}"
-      puts "host\t #{@url.host}"
-      puts "user\t #{@url.user}"
-      puts "port\t #{@url.port}"
-      puts "options\t #{@options.inspect}"
+      puts "host\t #{@target.host}"
+      puts "user\t #{@target.user}"
+      puts "port\t #{@target.port}"
       puts "---------------------------------------------------------------"
+    end
+
+    def chdir_or_create directory, mkdir = false
+    end
 
+    def remove! target
     end
 
     def close

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -8,12 +8,9 @@ module RestFtpDaemon
     # Class options
     attr_reader :ftp
 
-    def initialize url, log_prefix, options = {}
-      # Call super
-      super
-
+    def prepare
       # Create FTP object
-      if options[:ftpes]
+      if @ftpes
         prepare_ftpes
       else
         prepare_ftp
@@ -25,14 +22,14 @@ module RestFtpDaemon
       @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
 
       # Announce object
-      log_debug "RemoteFTP.initialize chunk_size:#{@chunk_size}"
+      log_debug "RemoteFTP.prepare chunk_size:#{@chunk_size}"
     end
 
     def connect
       # Connect remote server
       super
-      @ftp.connect @url.host, @url.port
-      @ftp.login @url.user, @url.password
+      @ftp.connect @target.host, @target.port
+      @ftp.login @target.user, @target.password
     end
 
     def present? target
@@ -88,7 +85,7 @@ module RestFtpDaemon
 
     def push source, target, tempname = nil, &callback
       # Push init
-      raise RestFtpDaemon::AssertionFailed, "push/1" if @ftp.nil?
+      raise RestFtpDaemon::AssertionFailed, "push/ftp" if @ftp.nil?
 
       # Temp file if provided
       destination = target.clone

data/lib/rest-ftp-daemon/remote_s3.rb

@@ -0,0 +1,52 @@
+require 'aws-sdk-resources'
+
+# Handle sFTP transfers for Remote class
+module RestFtpDaemon
+  class RemoteS3 < Remote
+
+    # Class options
+    attr_reader :client
+    attr_reader :target
+
+    def prepare
+      log_debug "RemoteS3.prepare target[#{@target.inspect}]"
+    end
+
+    def connect
+      # Connect init
+      super
+      log_debug "RemoteS3.connect [#{@target.aws_id}]@[#{@target.aws_bucket}]"
+
+      # Debug level
+      verbosity =  @debug ? Logger::DEBUG : false
+
+      # Connect remote server
+      @client = Aws::S3::Resource.new(
+        region: @target.aws_region,
+        credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret)
+        )
+    end
+
+    def push source, target, tempname = nil, &callback
+      # Push init
+      raise RestFtpDaemon::AssertionFailed, "push/client" if @client.nil?
+      log_debug "RemoteS3.push bucket[#{target.aws_bucket}] name[#{target.name}]"
+
+      # Do the transfer
+      bucket = @client.bucket(target.aws_bucket)
+      object = bucket.object(target.name)
+      object.put(body:'Hello World!')
+
+      # Dump information about this file
+      log_debug "RemoteS3.push url[#{object.public_url}]"
+      log_debug "RemoteS3.push etag[#{object.etag}]"
+      set_info :target, :aws_public_url, object.public_url
+      set_info :target, :aws_etag, object.etag
+    end
+
+    def connected?
+      !@client.nil?
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -7,27 +7,22 @@ module RestFtpDaemon
     # Class options
     attr_reader :sftp
 
-    def initialize url, log_prefix, options = {}
-      # Call super
-      super
-
-      # Announce object
-      log_debug "RemoteSFTP.initialize"
+    def prepare
     end
 
     def connect
       # Connect init
       super
-      log_debug "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
+      log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
 
       # Debug level
       verbosity =  @debug ? Logger::DEBUG : false
 
       # Connect remote server
-      @sftp = Net::SFTP.start(@url.host.to_s, @url.user.to_s,
-          password: @url.password.to_s,
+      @sftp = Net::SFTP.start(@target.host.to_s, @target.user.to_s,
+          password: @target.password.to_s,
           verbose: verbosity,
-          port: @url.port,
+          port: @target.port,
           non_interactive: true,
           timeout: DEFAULT_SFTP_TIMEOUT
           )
@@ -93,7 +88,7 @@ module RestFtpDaemon
 
     def push source, target, tempname = nil, &callback
       # Push init
-      raise RestFtpDaemon::AssertionFailed, "push/1" if @sftp.nil?
+      raise RestFtpDaemon::AssertionFailed, "push/sftp" if @sftp.nil?
 
       # Temp file if provided
       destination = target.clone

data/rest-ftp-daemon.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version                    = "0.410.5"
+  spec.version                    = "0.420.1"
 
   # Project description
   spec.name                       = "rest-ftp-daemon"
@@ -56,6 +56,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency     "net-sftp"
   spec.add_runtime_dependency     "double-bag-ftps"
+  spec.add_runtime_dependency     "aws-sdk-resources", '~> 2'
 
   spec.add_runtime_dependency     "streamio-ffmpeg"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.410.5
+  version: 0.420.1
 platform: ruby
 authors:
 - Bruno MEDICI
@@ -360,6 +360,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-resources
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: streamio-ffmpeg
   requirement: !ruby/object:Gem::Requirement
@@ -422,8 +436,10 @@ files:
 - lib/rest-ftp-daemon/metrics.rb
 - lib/rest-ftp-daemon/notification.rb
 - lib/rest-ftp-daemon/paginate.rb
+- lib/rest-ftp-daemon/pretty_json.rb
 - lib/rest-ftp-daemon/remote.rb
 - lib/rest-ftp-daemon/remote_ftp.rb
+- lib/rest-ftp-daemon/remote_s3.rb
 - lib/rest-ftp-daemon/remote_sftp.rb
 - lib/rest-ftp-daemon/static/config.json
 - lib/rest-ftp-daemon/static/css/bootstrap.min.css