rest-ftp-daemon 0.60 → 0.70

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWFjZTBkMGEwMWVhNjEzMmFhZDFhY2IyZmE0OThmYzk0YzdlNzAxMQ==
+    MDZkM2FiNjVjM2U0ZWYzOWFiOThhMGVjM2YzMjc2NzFmYWVkN2QyYQ==
   data.tar.gz: !binary |-
-    OTFlYWZiODY4MjY4OGFlOWY2NzI4Yjk4YmE4OWFhZjFiZmIxNzg1MQ==
+    YmVlMjAzMjEzYjg3NTkzNDQzYmM3NTM5MmE3OGQxODM3MTcwOTEzNA==
 SHA512:
   metadata.gz: !binary |-
-    OGIwMWE4NTQxM2IzOGRkM2E5MWY3YzdlYjI4N2EyMjdmNTYzYmJjOGY1NDUw
-    MGQ4MzAyN2ZiODY1N2FhOWZkOWZiMmMzMjQ2MmZjMGM1MDY0MjEyOWE1N2Ew
-    NGEwNjkyN2NjMGFjNWJkYzI4ODdjZjkwNGM1YzkzMjhkNjg4NTA=
+    MjYxNGE4N2YwNTI4YzhkMmJiODczMTYyZWYyOTI5YmVhNWJhZDkzOWUwOTg2
+    YWI4MTBkZjM1ZWVhODgwZWViZGFkNzMyMjg3YWVjNWI1NjhiMGMwNWJhMmMy
+    ZGE5NTNmMzM1YmY3OGI4NzdlMjU3MjYyNzlhMThlOWFlZWE0NDg=
   data.tar.gz: !binary |-
-    OTA1NDU1ZGU4NjNmNTE2Y2EzODM4MDE5OGU2M2RhOWRhNjM5ZDdiNzg4ZDIz
-    M2E4ZGJmNzYxZTk5MWQ3NDJmZGUyZTMwMDI3ODY4MTRmOWM0N2JmMDcyZjY1
-    ZmFiYTI5NTQ3OTY0OTBhZjE0YWM3NDBiNjI0MGNkM2FiYjhhMzc=
+    NjUwOWNlZDRiOWY3Mjk0OTI5N2MwODA0NmY5YTYxZmRiMTViZWE3ZDNjZGZi
+    YmViNGQ0M2FlNTI4M2EzZTE1OTIzZmZiMmJkNTFiMzUyYzg0MjYxNDQ0N2Y2
+    NWI0M2YxOTUyNjM3N2MwODRkNDI0ZjBmZWRiZGU2ODM3OWQ3NzI=

data/Gemfile.lock

@@ -1,11 +1,14 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.55)
+    rest-ftp-daemon (0.63)
+      double-bag-ftps
       facter
       grape
+      haml
       json
       settingslogic
+      sys-cpu
       thin (~> 1.6)
 
 GEM
@@ -28,10 +31,12 @@ GEM
     daemons (1.1.9)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
+    double-bag-ftps (0.1.2)
     equalizer (0.0.9)
     eventmachine (1.0.3)
     facter (2.2.0)
       CFPropertyList (~> 2.2.6)
+    ffi (1.9.5)
     grape (0.9.0)
       activesupport
       builder
@@ -42,11 +47,13 @@ GEM
       rack-accept
       rack-mount
       virtus (>= 1.0.0)
+    haml (4.0.5)
+      tilt
     hashie (3.3.1)
     i18n (0.6.11)
     ice_nine (0.11.0)
     json (1.8.1)
-    minitest (5.4.1)
+    minitest (5.4.2)
     multi_json (1.10.1)
     multi_xml (0.5.5)
     rack (1.5.2)
@@ -56,11 +63,14 @@ GEM
       rack (>= 1.0.0)
     rake (10.3.2)
     settingslogic (2.0.9)
-    thin (1.6.2)
-      daemons (>= 1.0.9)
-      eventmachine (>= 1.0.0)
-      rack (>= 1.0.0)
+    sys-cpu (0.7.1)
+      ffi (>= 1.0.0)
+    thin (1.6.3)
+      daemons (~> 1.0, >= 1.0.9)
+      eventmachine (~> 1.0)
+      rack (~> 1.0)
     thread_safe (0.3.4)
+    tilt (2.0.1)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     virtus (1.0.3)

data/README.md

@@ -18,7 +18,7 @@ As of today, its main features are :
 * Provide RESTful notifications to the requesting client
 * Allow authentication in FTP target in a standard URI-format
 * Allow configuration-based path templates to abstract local mounts or remote FTPs (endpoint tokens)
-
+* Remote supported protocols: FTP and FTPs
 
 Expected features in a short-time range :
 
@@ -28,7 +28,8 @@ Expected features in a short-time range :
 * Allow fallback file source when first file path is unavailable (failover)
 * Provide swagger-style API documentation
 * Authenticate API clients
-
+* Allow to specify random remote/local source/target
+* Allow more remote protocols: sFTP, FTPs, HTTP POST, etc
 
 
 Installation
@@ -39,7 +40,7 @@ This project is available as a rubygem, requires on ruby >= 1.9.3 and rubygems i
 Get and install the gem from rubygems.org:
 
 ```
-# apt-get install ruby1.9.3 ruby-dev rubygems gcc g++
+# apt-get install ruby1.9.3 ruby1.9.1-dev ruby-dev rubygems gcc g++
 gem install rest-ftp-daemon --no-ri --no-rdoc
 ```
 
@@ -55,10 +56,10 @@ Start the daemon on a specific port :
 rest-ftp-daemon -p4000 start
 ```
 
-Check that the daemon is running and giving status info
+Check that the daemon is running and providing its status info
 
 ```
-http://localhost:3000/
+http://localhost:3200/
 ```
 
 Configuration
@@ -155,7 +156,7 @@ curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localh
 Getting status
 ------------------------------------------------------------------------------------
 
-* A global JSON status is provided on ``` GET /index.json ```
+* A global JSON status is provided on ``` GET /status ```
 
 * A nice dashboard gives a global view of the daemon, jobs in queue, and system status, exposed on ``` GET /```
 

data/bin/rest-ftp-daemon

@@ -2,41 +2,40 @@
 
 # Libs and init
 require "thin"
-DEVELOPMENT=ARGV.include?("development")
-HERE=File.expand_path(File.dirname(__FILE__) + '/../')
-#APP_NAME2="rest-ftp-daemon"
 
 # Include config file and build rack commandline
-require File.expand_path("#{HERE}/lib/rest-ftp-daemon/config.rb")
+app_root = File.expand_path(File.dirname(__FILE__) + '/../')
+require File.expand_path("lib/rest-ftp-daemon/config.rb")
 
+# Build rackup context
+settings_daemonize = [nil, true, 1, "1"].include?(Settings.daemonize)
 argv = ARGV
-argv << ["-p", Settings.port.to_s] unless ARGV.include?("-p") || Settings.port.nil?
-argv << ["-e", Settings.namespace] unless ARGV.include?("-e") || Settings.namespace.nil?
-argv << ["-l", Settings.logs.thin] unless ARGV.include?("-l") unless Settings.logs.thin.nil? unless Settings.logs.nil?
-argv << ["--daemonize"] if (Settings.daemonize==1 || Settings.daemonize.nil?) unless ARGV.include?("-d")
-
-# Rackup file
-argv << ["-R", File.expand_path("#{HERE}/config.ru")] unless ARGV.include?("-R")
+argv << ["-e", "development"] unless ARGV.include?("-e")
+argv << ["-p", Settings[:port].to_s] unless ARGV.include?("-p") || Settings[:port].nil?
+argv << ["-l", Settings[:logs][:thin]] unless ARGV.include?("-l") unless Settings[:logs][:thin].nil? unless Settings[:logs].nil?
+argv << ["--daemonize"] if settings_daemonize unless (ARGV.include?("-d") || ARGV.include?("--daemonize"))
+argv << ["-R", File.expand_path("config.ru")] unless ARGV.include?("-R")
 
 # Display compiled configuration
-puts
-puts "Daemon name \t #{Settings.name}"
-puts "Version     \t #{Settings.version}"
+puts "---"
+puts "Daemon name \t #{Settings.app_name}"
+puts "Version     \t #{Settings.app_ver}"
 puts "Environment \t #{Settings.namespace}"
 puts "Config file \t #{Settings.source}"
 puts "Parameters  \t #{argv.flatten}"
 puts Settings.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion => false )
 
 # Start Thin with this rackup configuration
+puts "---"
 puts
 begin
   Thin::Runner.new(argv.flatten).run!
-rescue Thin::PidFileExist
-  puts "EXITING: daemon was already running (Thin::PidFileExist)"
-rescue Thin::PidFileNotFound
-  puts "EXITING: daemon was not running (Thin::PidFileNotFound)"
-else
-  puts "PROCESS ENDING"
+  rescue Thin::PidFileExist
+    puts "ERROR: daemon was already running (Thin::PidFileExist)"
+  rescue Thin::PidFileNotFound
+    puts "ERROR: daemon was not running (Thin::PidFileNotFound)"
+  rescue SystemExit
+    puts "EXITING: daemon in the background (SystemExit)"
+  else
+    puts "EXITING: process ending"
 end
-
-

data/config.ru

@@ -1,14 +1,23 @@
 # Load gem files
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
+APP_LIBS = File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
+$LOAD_PATH.unshift(APP_LIBS) unless $LOAD_PATH.include?(APP_LIBS)
 require 'rest-ftp-daemon'
 
-# Some extra constants
-APP_STARTED = Time.now
-
-# Create worker pool
+# Create queue and worker pool
 $queue = RestFtpDaemon::JobQueue.new
 $pool = RestFtpDaemon::WorkerPool.new(Settings.workers.to_i)
 
-# Start REST FTP Daemon
-use Rack::Static, :urls => ["/css", "/images"], :root => "lib/rest-ftp-daemon/static/"
+# Rack middleware
+# use Rack::Etag           # Add an ETag
+# use Rack::Reloader,   0
+# Rack::Auth::Basic
+# use Rack::Auth::Basic, "Restricted Area" do |username, password|
+#   [username, password] == ['admin', 'admin']
+# end
+#use Rack::Deflator      # Compress
+
+# Serve static assets
+use Rack::Static, :urls => ["/css", "/images"], :root => "lib/#{Settings.app_name}/static/"
+
+# Launch the main daemon
 run Rack::Cascade.new [RestFtpDaemon::API::Root]

data/lib/rest-ftp-daemon.rb

@@ -8,6 +8,7 @@ require 'securerandom'
 require 'rest-ftp-daemon/config'
 require 'rest-ftp-daemon/exceptions'
 require 'rest-ftp-daemon/common'
+require 'rest-ftp-daemon/uri'
 require 'rest-ftp-daemon/job_queue'
 require 'rest-ftp-daemon/worker_pool'
 require 'rest-ftp-daemon/logger'

data/lib/rest-ftp-daemon/api/defaults.rb

@@ -45,7 +45,6 @@ module RestFtpDaemon
         # end
 
         helpers do
-
           def format_nice_bytes( number )
             return "Ø" if number.nil? || number.zero?
             index = ( Math.log( number ) / Math.log( 2 ) ).to_i / 10
@@ -63,7 +62,7 @@ module RestFtpDaemon
           end
 
           def render name, values={}
-            template = File.read("#{APP_ROOT}/lib/#{APP_NAME}/views/#{name.to_s}.haml")
+            template = File.read("lib/#{Settings.app_name}/views/#{name.to_s}.haml")
             haml_engine = Haml::Engine.new(template)
             haml_engine.render(binding, values)
           end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -2,13 +2,33 @@ module RestFtpDaemon
   module API
 
     class Jobs < Grape::API
-      include RestFtpDaemon::API::Defaults
-      logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
+
+
+####### CLASS CONFIG
+
+      #include RestFtpDaemon::API::Defaults
+      #logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
 
       params do
         optional :overwrite, type: Integer, default: false
       end
 
+
+####### INITIALIZATION
+
+      def initialize
+        $last_worker_id = 0
+
+        # Check that Queue and Pool are available
+        raise RestFtpDaemon::MissingQueue unless defined? $queue
+        raise RestFtpDaemon::MissingQueue unless defined? $pool
+
+        super
+      end
+
+
+####### HELPERS
+
       helpers do
 
         def threads_with_id job_id
@@ -42,6 +62,9 @@ module RestFtpDaemon
 
       end
 
+
+####### API DEFINITION
+
       desc "Get information about a specific job"
       params do
         requires :id, type: Integer, desc: "job id"

data/lib/rest-ftp-daemon/api/root.rb

@@ -7,13 +7,19 @@ module RestFtpDaemon
   module API
 
     class Root < Grape::API
+
+
+####### CLASS CONFIG
       include RestFtpDaemon::API::Defaults
-      logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
+      logger ActiveSupport::Logger.new(Settings.logs.api, 'daily') unless Settings.logs.api.nil?
       #add_swagger_documentation
 
       mount RestFtpDaemon::API::Jobs => '/jobs'
       # mount RestFtpDaemon::API::Workers => '/workers'
 
+
+####### HELPERS
+
       helpers do
         def info message, level = 0
           Root.logger.add(Logger::INFO, "#{'  '*level} #{message}", "API::Root")
@@ -32,19 +38,21 @@ module RestFtpDaemon
 
       end
 
-      ######################################################################
-      ####### INIT
-      ######################################################################
+
+####### INITIALIZATION
+
       def initialize
         $last_worker_id = 0
+
+        # Check that Queue and Pool are available
+        raise RestFtpDaemon::MissingQueue unless defined? $queue
+        raise RestFtpDaemon::MissingQueue unless defined? $pool
+
         super
       end
 
 
-
-      ######################################################################
-      ####### API DEFINITION
-      ######################################################################
+####### API DEFINITION
 
       # Server global status
       get '/' do
@@ -90,15 +98,15 @@ module RestFtpDaemon
       end
 
       # Server global status
-      get '/index.json' do
+      get '/status' do
         info "GET /"
         status 200
         return  {
           hostname: `hostname`.chomp,
-          version: Settings.version,
+          version: Settings.app_ver,
           config: Settings.to_hash,
-          started: APP_STARTED,
-          uptime: (Time.now - APP_STARTED).round(1),
+          started: Settings.app_started,
+          uptime: (Time.now - Settings.app_started).round(1),
           status: job_list_by_status,
           queue_size: $queue.all_size,
           jobs_queued: $queue.queued.collect(&:id),

data/lib/rest-ftp-daemon/common.rb

@@ -18,7 +18,7 @@ module RestFtpDaemon
     def info message, level = 0
       # progname = "Job [#{id}]" unless id.nil?
       # progname = "Worker [#{id}]" unless worker_id.nil?
-      @logger.add(Logger::INFO, "#{'  '*(level+1)} #{message}", progname)
+      @logger.add(Logger::INFO, "#{'  '*(level+1)} #{message}", progname) unless @logger.nil?
     end
 
     def notify signal, error = 0, status = {}

data/lib/rest-ftp-daemon/config.rb

@@ -1,24 +1,24 @@
 require 'settingslogic'
-DEVELOPMENT = false unless defined? DEVELOPMENT
-APP_NAME="rest-ftp-daemon"
 
-class Settings < Settingslogic
-  namespace DEVELOPMENT ? "development" : "production"
-  suppress_errors namespace!="development"
-end
+# Terrific assertions
+#raise "config.rb: APP_ROOT is not defined" unless defined? APP_ROOT
+APP_NAME = "rest-ftp-daemon"
+APP_CONF = "/etc/#{APP_NAME}.yml"
+APP_DEV = ARGV.include?("development") ? true : false
 
-# Fix application defaults and load config file if found
-app_config_file = "/etc/#{APP_NAME}.yml"
-if File.exists? app_config_file
-  Settings.source app_config_file
-else
-  Settings.source Hash.new
-end
+class Settings < Settingslogic
+  # Read configuration
+  source (File.exists? APP_CONF) ? APP_CONF : Hash.new
+  namespace (APP_DEV ? "development" : "production")
+  suppress_errors true
 
-# Forced shared settings
-Settings[:name] = APP_NAME
-Settings[:version] = "0.60"
+  # Some constants
+  self[:dev] = APP_DEV
+  self[:app_name] = APP_NAME
+  self[:app_ver] = "0.70"
+  self[:app_started] = Time.now
 
-# Forced fixed settings
-Settings[:default_trim_progname] = "18"
-Settings[:default_chunk_size] = "1000000"
+  # Some defaults
+  self[:app_trim_progname] = "18"
+  self[:app_chunk_size] = "1000000"
+end

data/lib/rest-ftp-daemon/exceptions.rb

@@ -4,6 +4,10 @@ module RestFtpDaemon
 
   class DummyException < RestFtpDaemonException; end
 
+  class MissingQueue   < RestFtpDaemonException; end
+  class MissingPool   < RestFtpDaemonException; end
+
+
   class RequestSourceMissing     < RestFtpDaemonException; end
   class RequestSourceNotFound    < RestFtpDaemonException; end
   class RequestTargetMissing     < RestFtpDaemonException; end
@@ -15,9 +19,12 @@ module RestFtpDaemon
   class JobSourceMissing         < RestFtpDaemonException; end
   class JobSourceNotFound        < RestFtpDaemonException; end
   class JobTargetMissing         < RestFtpDaemonException; end
+  class JobTargetUnsupported     < RestFtpDaemonException; end
   class JobTargetUnparseable     < RestFtpDaemonException; end
   #class JobTargetPermission      < RestFtpDaemonException; end
   class JobTargetFileExists      < RestFtpDaemonException; end
+  class JobPrerequisitesNotMet      < RestFtpDaemonException; end
+
 
   class NotificationMissingUrl   < RestFtpDaemonException; end
   class NotificationMissingSignal   < RestFtpDaemonException; end

data/lib/rest-ftp-daemon/job.rb

@@ -1,5 +1,8 @@
-require 'net/ftp'
+#require 'net/ftptls'
 
+require 'uri'
+require 'net/ftp'
+require 'double_bag_ftps'
 
 module RestFtpDaemon
   class Job < RestFtpDaemon::Common
@@ -10,8 +13,6 @@ module RestFtpDaemon
 
       # Grab params
       @params = params
-      @target = nil
-      @source = nil
 
       # Init context
       set :id, id
@@ -57,22 +58,17 @@ module RestFtpDaemon
         set :error, exception.class
 
       rescue RestFtpDaemonException => exception
-        info "Job.process failed [RestFtpDaemonException::#{exception.class}]"
+        info "Job.process failed [::#{exception.class}]"
         set :status, :failed
         set :error, exception.class
 
-      # rescue Exception => exception
-      #   set :status, :crashed
-      #   set :error, exception.class
+      rescue Exception => exception
+        info "Job.process exception [#{exception.class}] #{exception.backtrace.inspect}"
+        set :status, :crashed
+        set :error, exception.class
 
       else
         info "Job.process finished"
-# set :error, 0
-        #set :status, :wandering
-
-        # Wait for a few seconds before marking the job as finished
-        # info "#{prefix} wander for #{RestFtpDaemon::THREAD_SLEEP_BEFORE_DIE} sec"
-        # wander RestFtpDaemon::THREAD_SLEEP_BEFORE_DIE
         set :status, :finished
       end
 
@@ -80,8 +76,8 @@ module RestFtpDaemon
 
     def describe
       # Update realtime info
-      w = wandering_time
-      set :wandering, w.round(2) unless w.nil?
+      #w = wandering_time
+      #set :wandering, w.round(2) unless w.nil?
 
       # Update realtime info
       u = up_time
@@ -131,15 +127,15 @@ module RestFtpDaemon
       @params[attribute.to_s] = value
     end
 
-    def expand_path_from path
-      File.expand_path replace_token_in_path(path)
+    def expand_path path
+      File.expand_path replace_token(path)
     end
 
-    def expand_url_from path
-      URI replace_token_in_path(path) rescue nil
+    def expand_url path
+      URI::parse replace_token(path) rescue nil
     end
 
-    def replace_token_in_path path
+    def replace_token path
       # Ensure endpoints are not a nil value
       return path unless Settings.endpoints.is_a? Enumerable
       newpath = path.clone
@@ -157,21 +153,36 @@ module RestFtpDaemon
 
     def prepare
       # Init
+      info "Job.prepare"
       set :status, :preparing
+      @source_method = :file
+      @target_method = nil
+      @source_path = nil
+      @target_url = nil
 
       # Check source
       raise JobSourceMissing unless @params["source"]
-      @source = expand_path_from @params["source"]
-      set :debug_source, @source
+      #@source = expand_path @params["source"]
+      @source_path = expand_path @params["source"]
+      set :source_path, @source_path
+      set :source_method, :file
 
       # Check target
       raise JobTargetMissing unless @params["target"]
-      @target = expand_url_from @params["target"]
-      set :debug_target, @target.inspect
+      @target_url = expand_url @params["target"]
+      set :target_url, @target_url.inspect
+
+      if @target_url.kind_of? URI::FTP
+        @target_method = :ftp
+      elsif @target_url.kind_of? URI::FTPS
+        @target_method = :ftps
+      end
+      set :target_method, @target_method
 
       # Check compliance
-      raise JobTargetUnparseable if @target.nil?
-      raise JobSourceNotFound unless File.exists? @source
+      raise JobTargetUnparseable if @target_url.nil?
+      raise JobTargetUnsupported if @target_method.nil?
+      raise JobSourceNotFound unless File.exists? @source_path
 
     end
 
@@ -187,44 +198,90 @@ module RestFtpDaemon
     end
 
     def transfer
+      # Init
+      info "Job.transfer"
+
       # Send first notification
       transferred = 0
       notify "rftpd.started"
 
       # Ensure @source and @target are there
+      info "Job.transfer checking_source"
       set :status, :checking_source
-      raise JobPrerequisitesNotMet unless @source
-      raise JobPrerequisitesNotMet unless @source
-      target_path = File.dirname @target.path
-      target_name = File.basename @target.path
+      raise RestFtpDaemon::JobPrerequisitesNotMet unless @source_path
+      raise RestFtpDaemon::JobPrerequisitesNotMet unless @target_url
+      target_path = File.dirname @target_url.path
+      target_name = File.basename @target_url.path
 
       # Read source file size
-      source_size = File.size @source
+      source_size = File.size @source_path
       set :file_size, source_size
 
       # Prepare FTP transfer
-      set :status, :checking_target
-      ftp = Net::FTP.new(@target.host)
+      info "Job.transfer preparing"
+
+      # Scheme-aware config
+      case @target_method
+      when :ftp
+        info "Job.transfer scheme FTP"
+        ftp = Net::FTP.new
+      when :ftps
+        info "Job.transfer scheme FTPS"
+        ftp = DoubleBagFTPS.new
+        ftp.ssl_context = DoubleBagFTPS.create_ssl_context(:verify_mode => OpenSSL::SSL::VERIFY_NONE)
+        ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
+      else
+        info "Job.transfer scheme other: [#{@target_url.scheme}]"
+      end
+
+      # Connect remote server
+      info "Job.transfer connecting"
+      set :status, :connecting
+      ftp.connect(@target_url.host)
       ftp.passive = true
-      ftp.login @target.user, @target.password
+
+      # Logging in
+      info "Job.transfer login"
+      begin
+        u = ftp.login @target_url.user, @target_url.password
+      rescue Exception => exception
+        info "Job.process login failed [#{exception.class}] #{u.inspect}"
+        set :status, :login_failed
+        set :error, exception.class
+      end
+
+      # Changing to directory
+      info "Job.transfer chdir"
+      set :status, :chdir
       ftp.chdir(target_path)
 
       # Check for target file presence
       if get(:overwrite).nil?
-        results = ftp.list(target_name)
-        #results = ftp.list()
-
-        unless results.count.zero?
+        info "Job.transfer remote_check (#{target_name})"
+        set :status, :remote_check
+
+        # Get file list, sometimes the response can be an empty value
+        results = ftp.list(target_name) rescue nil
+
+        # Result can be nil or a list of files
+        if results.nil? || results.count.zero?
+          info "Job.transfer remote_absent"
+          set :status, :remote_absent
+        else
+          info "Job.transfer remote_present"
+          set :status, :remote_present
           ftp.close
           notify "rftpd.ended", RestFtpDaemon::JobTargetFileExists
           raise RestFtpDaemon::JobTargetFileExists
         end
+
       end
 
       # Do transfer
+      info "Job.transfer uploading"
       set :status, :uploading
-      chunk_size = Settings.transfer.chunk_size || Settings[:default_chunk_size]
-      ftp.putbinaryfile(@source, target_name, chunk_size) do |block|
+      chunk_size = Settings.transfer.chunk_size || Settings[:app_chunk_size]
+      ftp.putbinaryfile(@source_path, target_name, chunk_size) do |block|
         # Update counters
         transferred += block.bytesize
 
@@ -235,6 +292,8 @@ module RestFtpDaemon
       end
 
       # Close FTP connexion
+      info "Job.transfer closing"
+      set :status, :disconnecting
       notify "rftpd.ended"
       set :progress, nil
       ftp.close

data/lib/rest-ftp-daemon/logger.rb

@@ -1,7 +1,7 @@
 class Logger
   def format_message(severity, timestamp, progname, msg)
     stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
-    progname = "%-#{Settings[:default_trim_progname]}s" % progname
+    progname = "%-#{Settings[:app_trim_progname]}s" % progname
     "#{stamp}  #{severity}  #{progname}  #{msg}\n"
   end
 end

data/lib/rest-ftp-daemon/uri.rb

@@ -0,0 +1,6 @@
+module URI
+  class FTPS < Generic
+    DEFAULT_PORT = 990
+  end
+  @@schemes['FTPS'] = FTPS
+end

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -30,9 +30,9 @@
 
         .navbar-header
           %h1
-            = Settings.name
+            = Settings.app_name
             %small
-              = "v#{Settings.version}"
+              = "v#{Settings.app_ver}"
               = "[#{Settings.namespace}]"
 
 
@@ -70,17 +70,7 @@
 
       = render :dashboard_jobs, {jobs: @jobs_all}
 
-      =# render :dashboard_jobs, {kind: "queued", jobs: @jobs_queued}
-
-
-      %h2 Endpoint tokens
-      - endpoints = Settings.endpoints || {}
-      - endpoints.each do |token, value|
-        .btn-group.btn-group-sm
-          .btn.btn-default.btn-danger= token
-          .btn.btn-default= value
-
-
+      = render :dashboard_tokens, {tokens: Settings.endpoints || {}}
 
       %br
       %br

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -35,6 +35,7 @@
     %th= "priority"
     %th= "source"
     %th= "target"
+    %th= "mode"
     %th= "size"
     %th= "status"
     %th= "error"
@@ -60,8 +61,13 @@
 
       %td= job.id
       %td= job.get :priority
-      %td= job.get :source
-      %td= job.get :target
+      %td{title: job.get(:source_path)}
+        = job.get :source
+      %td{title: job.get(:target_url)}
+        = job.get :target
+      %td
+        .label.label-info= job.get :source_method
+        .label.label-primary= job.get :target_method
       %td= format_nice_bytes(size)
       %td= status
       %td= error

data/lib/rest-ftp-daemon/views/dashboard_tokens.haml

@@ -0,0 +1,9 @@
+%h2 Endpoint tokens
+
+%table.table.table-striped.table-hover.table-condensed
+  - tokens.each do |token, value|
+    %tr
+      %td
+        %b= token
+      %td
+        = value

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -12,9 +12,11 @@ module RestFtpDaemon
       super()
 
       # Create worker threads
+      info "WorkerPool initializing with #{number_threads} workers"
       number_threads.times do
         Thread.new { run }
       end
+
     end
 
     def wait
@@ -35,7 +37,7 @@ module RestFtpDaemon
 
       begin
         loop do
-          info "worker [#{@wid}] waiting for a job... "
+          info "worker [#{@wid}] ready, waiting for a job"
 
           # Wait for a job to come into the queue
           job = $queue.pop

data/rest-ftp-daemon.gemspec

@@ -1,12 +1,13 @@
 # coding: utf-8
 
 # Libs
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+# APP_ROOT = File.expand_path(File.dirname(__FILE__) + '/../')
+APP_LIBS = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(APP_LIBS) unless $LOAD_PATH.include?(APP_LIBS)
 require 'rest-ftp-daemon/config'
 
 Gem::Specification.new do |spec|
-  spec.name = Settings[:name]
+  spec.name = Settings[:app_name]
   spec.date = Time.now.strftime("%Y-%m-%d")
   spec.authors = ["Bruno MEDICI"]
   spec.email = "rest-ftp-daemon@bmconseil.com"
@@ -18,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.version       = Settings[:version]
+  spec.version       = Settings[:app_ver]
 
   spec.required_ruby_version = '>= 1.9.3'
 
@@ -30,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "settingslogic"
   spec.add_runtime_dependency "haml"
   spec.add_runtime_dependency "json"
+  spec.add_runtime_dependency "double-bag-ftps"
   spec.add_runtime_dependency "facter"
   spec.add_runtime_dependency "sys-cpu"
 

data/rest-ftp-daemon.yml.sample

@@ -1,6 +1,6 @@
 defaults: &defaults
   daemonize: 1
-  port: 3000
+  port: 3200
   workers: 1
 
   transfer:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.60'
+  version: '0.70'
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-23 00:00:00.000000000 Z
+date: 2014-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: double-bag-ftps
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: facter
   requirement: !ruby/object:Gem::Requirement
@@ -165,9 +179,10 @@ files:
 - lib/rest-ftp-daemon/logger.rb
 - lib/rest-ftp-daemon/notification.rb
 - lib/rest-ftp-daemon/static/css/bootstrap.min.css
+- lib/rest-ftp-daemon/uri.rb
 - lib/rest-ftp-daemon/views/dashboard.haml
 - lib/rest-ftp-daemon/views/dashboard_jobs.haml
-- lib/rest-ftp-daemon/views/index.haml
+- lib/rest-ftp-daemon/views/dashboard_tokens.haml
 - lib/rest-ftp-daemon/worker_pool.rb
 - rest-ftp-daemon.gemspec
 - rest-ftp-daemon.yml.sample

data/lib/rest-ftp-daemon/views/index.haml

@@ -1,116 +0,0 @@
-!!! 5
-%html{:lang => "en"}
-  %head
-    %meta{:charset => "utf-8"}/
-    %link{ href:"//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css" , rel: "stylesheet"}
-
-    %title="#{Settings.name} dashboard"
-
-    :css
-
-      #footer {
-        padding: 15px 0;
-        _border-top: 1px solid silver;
-        }
-
-      .progress {
-        margin-bottom: 0;
-        }
-
-  %body
-
-    .container
-      %h1
-        = Settings.name
-        %small
-          = "v#{Settings.version}"
-          = "[#{Settings.namespace}]"
-
-
-      %h2 System status
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-warning Processors
-        .btn.btn-default= @info_procs
-
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-warning Load
-        .btn.btn-default= @info_load.round(1)
-
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-warning CPU
-        .btn.btn-default= "#{@info_norm} %"
-
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-warning Free
-        .btn.btn-default= @info_memfree
-
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-success IP
-        .btn.btn-default= @info_ipaddr
-
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-success Workers
-        .btn.btn-default= Settings.workers
-
-      .btn-group.btn-group-sm
-        .btn.btn-default.btn-info Transferred
-        .btn.btn-default= format_nice_bytes(@info_total_transferred)
-
-
-      %h2
-        Jobs on this system (#{@jobs_all})
-        %small
-          queued (#{@jobs_queued}) /  popped (#{@jobs_popped})
-
-      %table.table.table-striped.table-hover.table-condensed
-        %tr
-          %th= "ID"
-          %th= "priority"
-          %th= "source"
-          %th= "target"
-          %th= "size"
-          %th= "status"
-          %th= "error"
-          %th= "progress"
-          %th
-
-        - @jobs.each do |job|
-          - error = job.get :error
-          - status = job.get :status
-          - size = job.get :file_size
-          - progress = job.get :progress
-
-          - if error!=0 && !error.nil?
-            - trclass = "danger"
-          - elsif status == :uploading
-            - trclass = "info"
-          - elsif status == :finished
-            - trclass = "success"
-          - else
-            - trclass = "warning"
-
-          %tr{class: trclass}
-
-            %td= job.id
-            %td= job.get :priority
-            %td= job.get :source
-            %td= job.get :target
-            %td= format_nice_bytes(size)
-            %td= status
-            %td= error
-            %td
-              - unless progress.nil?
-                .progress
-                  .progress-bar{style:"width: #{progress}%;"}
-                    = format_nice_bytes job.get(:file_sent)
-            %td
-              - unless progress.nil?
-                = "#{progress} %"
-
-
-      %h2 Endpoint tokens
-      - endpoints = Settings.endpoints || {}
-      - endpoints.each do |token, value|
-        .btn-group.btn-group-sm
-          .btn.btn-default.btn-danger= token
-          .btn.btn-default= value