rest-ftp-daemon 0.222.0 → 0.230.0

data/lib/rest-ftp-daemon/job_queue.rb

@@ -165,14 +165,16 @@ module RestFtpDaemon
       @waiting.size
     end
 
-    def expire status, maxage
+    def expire status, maxage, verbose = false
 # FIXME: clean both @jobs and @queue
       # Init
       return if status.nil? || maxage <= 0
 
       # Compute oldest possible birthday
       before = Time.now - maxage.to_i
-      # info "conchita_clean: SELECT status[#{status.to_s}] before[#{before}]"
+
+      # Verbose output ?
+      log_info "JobQueue.expire \t[#{status.to_s}] \tbefore \t[#{before}]" if verbose
 
       @mutex.synchronize do
         # Delete jobs from the queue when they match status and age limits
@@ -185,7 +187,7 @@ module RestFtpDaemon
 
           # Ok, we have to clean it up ..
           log_info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
-          log_info "       & unqueued" if @queue.delete(job)
+          log_info "#{LOG_INDENT}unqueued" if @queue.delete(job)
 
           true
         end

data/lib/rest-ftp-daemon/logger.rb

@@ -22,9 +22,9 @@ class Logger
 
     end
 
-
     # Prepend plain message to output
-    output.unshift (prefix1 + message.strip)
+    #output.unshift (prefix1 + message.strip)
+    output.unshift (prefix1 + message)
 
     # Send all this to logger
     add context[:level], output
@@ -43,7 +43,8 @@ class Logger
 
     def build_from_array prefix, lines
       lines.map do |value|
-        text = value.to_s.strip[0..LOG_TRIM_LINE]
+        #text = value.to_s.strip[0..LOG_TRIM_LINE]
+        text = value.to_s[0..LOG_TRIM_LINE]
         "#{prefix}#{text}"
       end
     end

data/lib/rest-ftp-daemon/logger_helper.rb

@@ -4,23 +4,27 @@ module RestFtpDaemon
   protected
 
     def log_info message, lines = []
-      logger.info_with_id message, log_context.merge({
-        from: self.class.to_s,
-        lines: lines,
-        level: Logger::INFO
-        })
+      log message, lines, Logger::INFO
     end
 
     def log_error message, lines = []
-      logger.info_with_id message, log_context.merge({
-        from: self.class.to_s,
-        lines: lines,
-        level: Logger::ERROR
-        })
+      log message, lines, Logger::ERROR
     end
 
     def log_context
       {}
     end
+
+  private
+
+    def log message, lines, level
+      context = log_context || {}
+      logger.info_with_id message, context.merge({
+        from: self.class.to_s,
+        lines: lines,
+        level: level
+        })
+    end
+
   end
 end

data/lib/rest-ftp-daemon/notification.rb

@@ -12,6 +12,10 @@ module RestFtpDaemon
     attr_accessor :job
 
     def initialize url, params
+      # Remember params
+      @url = url
+      @params = params
+
       # Generate a random key
       @id = Helpers.identifier(NOTIFY_IDENTIFIER_LEN)
       @jid = nil
@@ -19,64 +23,71 @@ module RestFtpDaemon
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :notify
 
+      # Handle the notification
+      log_info "initialized [#{@url}]"
+      process
+    end
+
+  protected
+
+    def process
       # Check context
-      if url.nil?
-        log_info "skipping (missing url): #{params.inspect}"
+      if @url.nil?
+        log_info "skipping (missing url): #{@params.inspect}"
         return
-      elsif params[:event].nil?
-        log_info "skipping (missing event): #{params.inspect}"
+      elsif @params[:event].nil?
+        log_info "skipping (missing event): #{@params.inspect}"
         return
       end
 
       # Build body and extract job ID if provided
-      body = {
-        id:       params[:id].to_s,
-        signal:   "#{NOTIFY_PREFIX}.#{params[:event]}",
-        error:    params[:error],
+      flags = {
+        id:       @params[:id].to_s,
+        signal:   "#{NOTIFY_PREFIX}.#{@params[:event]}",
+        error:    @params[:error],
         host:     Settings.host.to_s,
         }
-      body[:status] = params[:status] if params[:status].is_a? Enumerable
-      body[:message] = params[:message].to_s unless params[:message].nil?
-      @jid = params[:id]
-      log_info "initialized"
+      flags[:status] = @params[:status] if @params[:status].is_a? Enumerable
+      flags[:message] = @params[:message].to_s unless @params[:message].nil?
+      @jid = @params[:id]
 
-
-      # Send message in a thread
+      # Spawn a dedicated thread
       Thread.new do
-        # Prepare query
-        uri = URI(url)
-        headers = {
-          "Content-Type"  => "application/json",
-          "Accept"        => "application/json",
-          "User-Agent"    => "#{APP_NAME} - #{APP_VER}"
-           }
-        data = body.to_json
-        log_info "sending #{data}"
-
-
-        # Prepare HTTP client
-        http = Net::HTTP.new uri.host, uri.port
-        # http.initialize_http_header({'User-Agent' => APP_NAME})
-
-        # Post notification
-        response = http.post uri.path, data, headers
-
-        # Handle server response / multi-lines
-        response_lines = response.body.lines
-
-        if response_lines.size > 1
-          human_size = Helpers.format_bytes(response.body.bytesize, "B")
-          #human_size = 0
-          log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
-        else
-          log_info "received [#{response.code}] #{response.body.strip}"
-        end
+        send flags
+      end # end Thread
+    end
 
+    def send flags
+      # Prepare query
+      headers = {
+        "Content-Type"  => "application/json",
+        "Accept"        => "application/json",
+        "User-Agent"    => "#{APP_NAME} - #{APP_VER}"
+         }
+      data = flags.to_json
+
+      # Send notification through HTTP
+      uri = URI @url
+      http = Net::HTTP.new uri.host, uri.port
+
+      # Post notification, handle server response / multi-lines
+      log_info "sending #{data}"
+      response = http.post uri.path, data, headers
+      response_lines = response.body.lines
+
+      if response_lines.size > 1
+        human_size = Helpers.format_bytes(response.body.bytesize, "B")
+        log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
+      else
+        log_info "received [#{response.code}] #{response.body.strip}"
       end
 
+      # Handle exceptions
+      rescue StandardError => ex
+        log_error "EXCEPTION: #{ex.inspect}"
+
     end
 
-  protected
 
     def log_context
       {

data/lib/rest-ftp-daemon/paginate.rb

@@ -18,7 +18,7 @@ module RestFtpDaemon
       @total = @data.count
 
       # Count pages
-      @pages = (@total.to_f / PAGINATE_MAX).ceil
+      @pages = (@total.to_f / DEFAULT_PAGE_SIZE).ceil
       @pages = 1 if @pages < 1
     end
 
@@ -39,7 +39,7 @@ module RestFtpDaemon
     end
 
     def subset
-      size = PAGINATE_MAX.to_i
+      size = DEFAULT_PAGE_SIZE.to_i
       offset = (@page-1) * size
       @data[offset, size]
     end

data/lib/rest-ftp-daemon/path.rb

@@ -0,0 +1,43 @@
+module RestFtpDaemon
+  class Path
+    attr_accessor :name
+    attr_accessor :dir
+
+    def initialize full, strip_leading_slash = false
+      # Extract path parts
+      @name = extract_filename full.to_s
+      @dir = extract_dirname full.to_s
+
+      # Remove leading slash if needed
+      strip_leading_slash_from_dir! if strip_leading_slash
+    end
+
+    def full
+      return @name if @dir.nil? || @dir.empty?
+      return File.join @dir, @name
+    end
+
+    def size
+      File.size full if File.exists? full
+    end
+
+  private
+
+    def extract_filename path
+      # match everything that's after a slash at the end of the string
+      m = path.match /\/?([^\/]+)$/
+      return m[1].to_s unless m.nil?
+    end
+
+    def extract_dirname path
+      # match all the beginning of the string up to the last slash
+      m = path.match(/^(.*)\/[^\/]*$/)
+      return m[1].to_s unless m.nil?
+    end
+
+    def strip_leading_slash_from_dir!
+      @dir.to_s.gsub!(/^\//, '')
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/remote.rb

@@ -0,0 +1,57 @@
+module RestFtpDaemon
+  class Remote
+    include LoggerHelper
+    attr_reader :logger
+    attr_reader :log_context
+
+    def initialize url, log_context, options = {}
+      # Logger
+      @log_context = log_context || {}
+      @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
+
+      # Extract URL parts
+      @url = url
+      @url.user ||= "anonymous"
+
+      # Annnounce object
+      log_info "Remote.initialize [#{url.to_s}]"
+    end
+
+    def connect
+      # Debug mode ?
+      debug_header if @debug
+    end
+
+    def close
+      # Debug mode ?
+      puts "-------------------- SESSION CLOSING --------------------------" if @debug
+    end
+
+  private
+
+    # def log_context
+    #   @log_context
+    # end
+
+    def myname
+      self.class.to_s
+    end
+
+    def debug_header
+      # Output header to STDOUT
+      puts
+      puts "-------------------- SESSION STARTING -------------------------"
+      #puts "job id\t #{@id}"
+      #puts "source\t #{@source}"
+      #puts "target\t #{@target}"
+      puts "class\t #{myname}"
+      #puts "class\t #{myname}"
+      puts "host\t #{@url.host}"
+      puts "user\t #{@url.user}"
+      puts "port\t #{@url.port}"
+      puts "options\t #{@options.inspect}"
+      puts "---------------------------------------------------------------"
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -0,0 +1,141 @@
+module RestFtpDaemon
+  class RemoteFTP < Remote
+    attr_reader :ftp
+
+    def initialize url, log_context, options = {}
+      # Call super
+      super
+
+      # Use debug ?
+      @debug = (Settings.at :debug, :ftp) == true
+
+      # Create FTP object
+      if options[:ftpes]
+        prepare_ftpes
+      else
+        prepare_ftp
+      end
+      @ftp.passive = true
+      @ftp.debug_mode = !!@debug
+
+      # Config
+      @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
+
+      # Announce object
+      log_info "RemoteFTP.initialize chunk_size:#{@chunk_size}"
+    end
+
+    def connect
+      # Connect init
+      super
+
+      # Connect remote server
+      @ftp.connect @url.host, @url.port
+      @ftp.login @url.user, @url.password
+    end
+
+    def present? target
+      size = @ftp.size target.full
+      log_info "RemoteFTP.present? [#{target.name}]"
+
+      rescue Net::FTPPermError
+        # log_info "RemoteFTP.present? [#{target.name}] NOT_FOUND"
+        return false
+      else
+        return size
+    end
+
+    def remove! target
+      log_info "RemoteFTP.remove! [#{target.name}]"
+      @ftp.delete target.full
+      rescue Net::FTPPermError
+        log_info "#{LOG_INDENT}[#{target.name}] file not found"
+      else
+        log_info "#{LOG_INDENT}[#{target.name}] removed"
+    end
+
+    def mkdir directory
+      log_info "RemoteFTP.mkdir [#{directory}]"
+      @ftp.mkdir directory
+
+      rescue
+        raise JobTargetPermissionError
+    end
+
+    def chdir_or_create directory, mkdir = false
+      # Init, extract my parent name and my own name
+      log_info "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+      parent, current = Helpers.extract_parent(directory)
+
+      fulldir = "/#{directory}"
+
+      # Access this directory
+      begin
+        @ftp.chdir "/#{directory}"
+
+      rescue Net::FTPPermError => e
+        # If not allowed to create path, that's over, we're stuck
+        return false unless mkdir
+
+        #log_info "#{LOG_INDENT}upward [#{parent}]"
+        chdir_or_create parent, mkdir
+
+        # Now I was able to chdir into my parent, create the current directory
+        #log_info "#{LOG_INDENT}mkdir [#{directory}]"
+        mkdir "/#{directory}"
+
+        # Finally retry the chdir
+        retry
+      else
+        return true
+      end
+    end
+
+    def push source, target, tempname = nil, &callback
+      # Push init
+      raise RestFtpDaemon::JobAssertionFailed, "push/1" if @ftp.nil?
+
+      # Temp file if provided
+      destination = target.clone
+      destination.name = tempname if tempname
+
+      # Do the transfer
+      log_info "RemoteFTP.push to [#{destination.name}]"
+
+      @ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
+        # Update the worker activity marker
+        #FIXME worker_is_still_active
+
+        # Update job status after this block transfer
+        yield data.bytesize, destination.name
+      end
+
+    end
+
+    def close
+      # Close init
+      super
+
+      # Close FTP connexion and free up memory
+      @ftp.close
+    end
+
+    def connected?
+      !@ftp.welcome.nil?
+    end
+
+  private
+
+    def prepare_ftp
+      @ftp = Net::FTP.new
+    end
+
+   def prepare_ftpes
+      @ftp = DoubleBagFTPS.new
+      @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
+      @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
+    end
+
+
+  end
+end