rest-ftp-daemon 0.300.3 → 0.302.0

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -1,3 +1,5 @@
+require "grape"
+
 module RestFtpDaemon
   module API
     class Jobs < Grape::API

data/lib/rest-ftp-daemon/api/root.rb

@@ -1,3 +1,5 @@
+require "grape"
+
 module RestFtpDaemon
   module API
     class Root < Grape::API

data/lib/rest-ftp-daemon/api/status.rb

@@ -1,3 +1,4 @@
+require "grape"
 require "get_process_mem"
 
 module RestFtpDaemon

data/lib/rest-ftp-daemon/constants.rb

@@ -6,7 +6,7 @@ DEFAULT_POOL            = "default"
 DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
 DEFAULT_FTP_CHUNK       = 1024  # 1 MB
 DEFAULT_PAGE_SIZE       = 50    # 50 lines
-DEFAULT_RETRY_DELAY     = 10    # 10s
+DEFAULT_RETRY_AFTER     = 10    # 10s
 
 
 # Internal job constants
@@ -21,7 +21,7 @@ LOG_ROTATION             = "daily"
 LOG_FORMAT_PROGNAME     = "%d\t%s"
 
 LOG_HEADER_TIME          = "%Y-%m-%d %H:%M:%S"
-LOG_HEADER_FORMAT        = "%s \t%d\t%-8s %-15s "
+LOG_HEADER_FORMAT        = "%s \t%d\t%-8s %-10s "
 LOG_MESSAGE_TRIM         = 200
 LOG_MESSAGE_TEXT         = "%s%s"
 LOG_MESSAGE_ARRAY        = "%s     - %s"
@@ -29,9 +29,9 @@ LOG_MESSAGE_HASH         = "%s     * %-20s %s"
 
 # Constants: logger app-specific prefix
 LOG_PREFIX_WID           = 8
-LOG_PREFIX_JID           = JOB_IDENT_LEN + 3 + 2
-LOG_PREFIX_ID            = 6
-LOG_PREFIX_FORMAT        = "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s"
+LOG_PREFIX_JID           = JOB_IDENT_LEN + 4
+LOG_PREFIX_ID            = 5
+LOG_PREFIX_FORMAT        = "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s "
 
 
 # Constants: logger to be cleaned up

data/lib/rest-ftp-daemon/job.rb

@@ -24,6 +24,8 @@ module RestFtpDaemon
     attr_reader :infos
     attr_reader :pool
 
+    attr_accessor :config
+
     FIELDS.each do |name|
       attr_reader name
     end
@@ -43,8 +45,13 @@ module RestFtpDaemon
       @runs = 0
       @wid = nil
 
+      # Prepare configuration
+      @config       = Conf[:transfer] || {}
+      @endpoints    = Conf[:endpoints] || {}
+      @pools        = Conf[:pools] || {}
+
       # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
+      @logger = RestFtpDaemon::LoggerPool.instance.get :transfer
 
       # Protect with a mutex
       @mutex = Mutex.new
@@ -54,34 +61,24 @@ module RestFtpDaemon
         instance_variable_set "@#{name}", params[name]
       end
 
-      # Set pool
-      pools = (Conf[:pools] || {})
       # Check if pool name exists
-      if (pools.keys.include? params[:pool])
+      if (@pools.keys.include? params[:pool])
         @pool = params[:pool].to_s
       else
         @pool = DEFAULT_POOL
       end
 
       # Set job queue, thus reset
-      reset
-
-      # Read source file size and parameters
-      @notify_after_sec = Conf.at(:transfer, :notify_after_sec) rescue nil
+      # reset
     end
 
     def reset
-      # Set super-default flags
-      flag_default :mkdir, false
-      flag_default :overwrite, false
-      flag_default :tempfile, false
-
       # Flag current job
       @queued_at = Time.now
       @updated_at = Time.now
 
       # Send first notification
-      log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] interval[#{JOB_UPDATE_INTERVAL}]"
+      log_info "Job.initialize notify[queued]}]"
       client_notify :queued
 
       # Update job status
@@ -91,8 +88,7 @@ module RestFtpDaemon
     end
 
     def process
-      # Update job's status
-      log_info "Job.process"
+      log_info "Job.process update_interval[#{JOB_UPDATE_INTERVAL}]"
 
       # Prepare job
       begin
@@ -259,10 +255,6 @@ module RestFtpDaemon
 
   protected
 
-    def expand_path path
-      File.expand_path replace_tokens(path)
-    end
-
     def expand_url path
       URI.parse replace_tokens(path)
     end
@@ -273,8 +265,8 @@ module RestFtpDaemon
 
     def replace_tokens path
       # Ensure endpoints are not a nil value
-      return path unless Conf[:endpoints].is_a? Enumerable
-      vectors = Conf[:endpoints].clone
+      return path unless @endpoints.is_a? Enumerable
+      vectors = @endpoints.clone
 
       # Stack RANDOM into tokens
       vectors["RANDOM"] = SecureRandom.hex(JOB_RANDOM_LEN)
@@ -294,16 +286,21 @@ module RestFtpDaemon
     end
 
     def prepare
+      # Init
+      @source_path = nil
+
+      # Prepare flags
+      flag_prepare :mkdir, false
+      flag_prepare :overwrite, false
+      flag_prepare :tempfile, true
+
       # Update job status
       set_status JOB_STATUS_PREPARING
       @runs += 1
 
-      # Init
-      @source_path = nil
-
       # Prepare source
       raise RestFtpDaemon::JobMissingAttribute unless @source
-      @source_path = expand_path @source
+      @source_path = File.expand_path replace_tokens(@source)
       set_info :source, :path, @source_path
       set_info :source, :method, JOB_METHOD_FILE
 
@@ -324,21 +321,21 @@ module RestFtpDaemon
         # set_info :target, :method, :ftp
         set_info :target, :method, JOB_METHOD_FTP
         #@target_method = :ftp
-        @remote = RemoteFTP.new target_uri, log_prefix
+        @remote = RemoteFTP.new target_uri, log_prefix, debug: @config[:debug_ftp]
 
       elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
         log_info "Job.prepare target_method FTPES"
         # set_info :target, :method, :ftpes
         set_info :target, :method, JOB_METHOD_FTPS
         # @target_method = :ftpes
-        @remote = RemoteFTP.new target_uri, log_prefix, ftpes: true
+        @remote = RemoteFTP.new target_uri, log_prefix, debug: @config[:debug_sftp], ftpes: true
 
       elsif target_uri.is_a? URI::SFTP
         log_info "Job.prepare target_method SFTP"
         # set_info :target, :method, :sftp
         set_info :target, :method, JOB_METHOD_SFTP
         # @target_method = :sftp
-        @remote = RemoteSFTP.new target_uri, log_prefix
+        @remote = RemoteSFTP.new target_uri, log_prefix, debug: @config[:debug_sftp]
 
       else
         log_info "Job.prepare unknown scheme [#{target_uri.scheme}]"
@@ -460,15 +457,17 @@ module RestFtpDaemon
       touch_job
     end
 
-    def flag_default name, default
+    def flag_prepare name, default
       # build the flag instance var name
       variable = "@#{name}"
 
-      # If it's true or false, that's ok
-      return if [true, false].include? instance_variable_get(variable)
+      [config[name], default].each do |alt_value|
+        # If it's already true or false, that's ok
+        return if [true, false].include? instance_variable_get(variable)
 
-      # Otherwise, set it to the default value
-      instance_variable_set variable, default
+        # Otherwise, set it to the new alt_value
+        instance_variable_set variable, alt_value
+      end
     end
 
     def finalize
@@ -539,6 +538,7 @@ module RestFtpDaemon
     def progress transferred, name = ""
       # What's current time ?
       now = Time.now
+      notify_after = @config[:notify_after]
 
       # Update counters
       @transfer_sent += transferred
@@ -561,7 +561,7 @@ module RestFtpDaemon
         stack << (Helpers.format_bytes @transfer_total, "B")
         stack << (Helpers.format_bytes @current_bitrate.round(0), "bps")
         stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
-        log_debug "Job.progress #{stack2} \t#{name}"
+        log_debug "progress #{stack2} \t#{name}"
 
         # Remember when we last did it
         @progress_at = now
@@ -569,13 +569,13 @@ module RestFtpDaemon
 
       # Notify if requested
       notified_ago = (now.to_f - @notified_at.to_f)
-      if (!@notify_after_sec.nil?) && (notified_ago > @notify_after_sec)
+      if (!notify_after.nil?) && (notified_ago > notify_after)
         # Prepare and send notification
         notif_status = {
           progress: percent0,
           transfer_sent: @transfer_sent,
           transfer_total: @transfer_total,
-          transfer_bitrate: @current_bitrate,
+          transfer_bitrate: @current_bitrate.round(0),
           }
         client_notify :progress, status: notif_status
 

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -1,37 +1,60 @@
 require "logger"
+require "singleton"
 
+# Logger interface class to access logger though symbolic names
 module RestFtpDaemon
-
-  # Logger interface class to access logger though symbolic names
   class LoggerPool
     include Singleton
 
-    def initialize
-      @loggers = {}
-    end
-
     def get pipe
+      @loggers ||= {}
       @loggers[pipe] ||= create(pipe)
     end
 
     def create pipe
-      # Compute file path / STDERR
-      logfile = Conf[:logs][pipe] if Conf[:logs].is_a? Hash
-      logfile ||= STDERR
-
-      # Check if we can write to than file
-      unless File.writable?(logfile)
-        puts "LoggerPool [#{pipe}] logging to hyperspace (cannot write to #{logfile})"
-        logfile = nil
-      end
+      # Compute logfile or STDERR, and declare what we're doing
+      filename = logfile(pipe)
 
       # Create the logger and return it
-      logger = Logger.new(logfile, LOG_ROTATION)   #, 10, 1024000)
+      logger = Logger.new(filename, LOG_ROTATION)   #, 10, 1024000)
       logger.progname = pipe.to_s.downcase
       logger.formatter = Shared::LoggerFormatter
 
       # Finally return this logger
       logger
+
+    rescue Errno::EACCES
+      puts "LoggerPool [#{pipe}] failed: access error"
+    end
+
+  protected
+
+    def logfile pipe
+      # Disabled if no valid config
+      return nil unless Conf[:logs].is_a?(Hash)
+
+      # Compute logfile and check if we can write there
+      logfile = File.expand_path(Conf[:logs][pipe], Conf[:logs][:path])
+
+      # Check that we'll be able to create logfiles
+      if File.exists?(logfile)
+        # File is there, is it writable ?
+        unless File.writable?(logfile)
+          puts "LoggerPool [#{pipe}] disabled: file not writable [#{logfile}]"
+          return nil
+        end
+      else
+        # No file here, can we create it ?
+        logdir = File.dirname(logfile)
+        unless File.writable?(logdir)
+          puts "LoggerPool [#{pipe}] disabled: directory not writable [#{logdir}]"
+          return nil
+        end
+      end
+
+      # OK, return a clean file path
+      puts "LoggerPool [#{pipe}] logging to [#{logfile}]"
+      return logfile
     end
 
   end

data/lib/rest-ftp-daemon/notification.rb

@@ -1,3 +1,6 @@
+require 'api_auth'
+require 'rest_client'
+
 module RestFtpDaemon
 
   # Handles a notification POST using a dedicated thread
@@ -61,20 +64,24 @@ module RestFtpDaemon
 
     def send flags
       # Prepare query
-      headers = {
-        "Content-Type"  => "application/json",
-        "Accept"        => "application/json",
-        "User-Agent"    => "#{Conf.app_name}/v#{Conf.app_ver}"
-         }
-      data = flags.to_json
-
-      # Send notification through HTTP
       uri = URI @url
-      http = Net::HTTP.new uri.host, uri.port
-
-      # Post notification, handle server response / multi-lines
-      log_info "sending #{data}"
-      response = http.post uri.path, data, headers
+      # uri = URI(rule[:relay])
+      #http = Net::HTTP.new uri.host, uri.port
+
+      # Prepare request
+      request = RestClient::Request.new url: uri.to_s,
+        method: :post,
+        payload: JSON.pretty_generate(flags),
+        headers: {
+          content_type: :json,
+          accept: :json,
+          user_agent: Conf.generate(:user_agent),
+          }
+
+      # Execure request
+      log_info "posting #{flags.to_json}"
+      # response = http.post uri.path, data, headers
+      response = request.execute
 
       # Log reponse body
       response_lines = response.body.lines
@@ -86,14 +93,14 @@ module RestFtpDaemon
       end
 
       # Handle exceptions
-      rescue Net::OpenTimeout => ex
-        log_error "Net::OpenTimeout: #{ex.inspect}"
+      rescue Net::OpenTimeout => e
+        log_error "Net::OpenTimeout: #{e.message}"
 
-      rescue SocketError => ex
-        log_error "SocketError: #{ex.inspect}"
+      rescue SocketError => e
+        log_error "SocketError: #{e.message}"
 
-      rescue StandardError => ex
-        log_error "UNHANDLED EXCEPTION: #{ex.inspect}"
+      rescue StandardError => e
+        log_error "UNHANDLED EXCEPTION: #{e.message}", e.backtrace
     end
 
     def log_prefix

data/lib/rest-ftp-daemon/remote.rb

@@ -7,9 +7,12 @@ module RestFtpDaemon
     attr_reader :log_prefix
 
     def initialize url, log_prefix, options = {}
+      # Options
+      @debug = !!options[:debug]
+
       # Logger
       @log_prefix = log_prefix || {}
-      @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
+      @logger = RestFtpDaemon::LoggerPool.instance.get :transfer
 
       # Extract URL parts
       @url = url
@@ -21,12 +24,22 @@ module RestFtpDaemon
 
     def connect
       # Debug mode ?
-      debug_header if @debug
+      return unless @debug
+      puts
+      puts "-------------------- SESSION STARTING -------------------------"
+      puts "class\t #{myname}"
+      puts "host\t #{@url.host}"
+      puts "user\t #{@url.user}"
+      puts "port\t #{@url.port}"
+      puts "options\t #{@options.inspect}"
+      puts "---------------------------------------------------------------"
+
     end
 
     def close
       # Debug mode ?
-      puts "-------------------- SESSION CLOSING --------------------------" if @debug
+      return unless @debug
+      puts "-------------------- SESSION CLOSING --------------------------"
     end
 
   private
@@ -35,17 +48,5 @@ module RestFtpDaemon
       self.class.to_s
     end
 
-    def debug_header
-      # Output header to STDOUT
-      puts
-      puts "-------------------- SESSION STARTING -------------------------"
-      puts "class\t #{myname}"
-      puts "host\t #{@url.host}"
-      puts "user\t #{@url.user}"
-      puts "port\t #{@url.port}"
-      puts "options\t #{@options.inspect}"
-      puts "---------------------------------------------------------------"
-    end
-
   end
 end

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -10,9 +10,6 @@ module RestFtpDaemon
       # Call super
       super
 
-      # Use debug ?
-      @debug = (Conf.at :debug, :ftp) == true
-
       # Create FTP object
       if options[:ftpes]
         prepare_ftpes
@@ -20,7 +17,7 @@ module RestFtpDaemon
         prepare_ftp
       end
       @ftp.passive = true
-      @ftp.debug_mode = !!@debug
+      @ftp.debug_mode =  @debug
 
       # Config
       @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
@@ -30,10 +27,8 @@ module RestFtpDaemon
     end
 
     def connect
-      # Connect init
-      super
-
       # Connect remote server
+      super
       @ftp.connect @url.host, @url.port
       @ftp.login @url.user, @url.password
     end