rest-ftp-daemon 0.101 → 0.103.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b3c42331f0e483617809300fcc7b6623e94e617b
-  data.tar.gz: 0184e556177357a18e19a686d3fc3ae852dcbffd
+  metadata.gz: c059a568b0a22503a734c966807a19c8ff426333
+  data.tar.gz: a3d2443af653af100e642e86ba7144a2b9985495
 SHA512:
-  metadata.gz: ead761e0002d4cfda4409fd43dd1c5828040b204fadb2286898c31ccd428edc41be0dd3c83a334d4c0c35770c5bd77856c3665279b539ca3614460990e5b78c6
-  data.tar.gz: 0518b79a4037e122c99d6ce50e660724f1db21693c0e5abfeeab319f6759dfce66461aaf7bc999500e967f833c67660fbadfa6a00f241be65c6b154099021ab7
+  metadata.gz: 07afa716bc8b3b701377cfe1a9452e22238cc8e82af287f544e499cc2a4d263bd266e3d83f8a2aa53ca37381653d1fff961bdbff6bf9446167cf1276fb8bddf8
+  data.tar.gz: 31fb4ea27d1e64278563eb3e2e93375c24d1f9794e632acd7afbeb897dc0a6d88024fe2dd3c4393648fa151b1f54c35432323f45cd90b4f9d368feeb208078a1

data/README.md

@@ -2,7 +2,7 @@ rest-ftp-daemon
 ====================================================================================
 
 
-This is a pretty simple FTP client daemon, controlled through a RESTfull API.
+This is a pretty simple FTP client daemon, controlled through a RESTful API.
 
 API documentation is [maintained on Apiary](http://docs.restftpdaemon.apiary.io/)
 

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -6,11 +6,8 @@ module RestFtpDaemon
         # Job ID
         expose :id
 
-        # Job specific attributes
-        Job::FIELDS.each do |field|
-          expose field
-          #expose field, unless: lambda { |object, options| object.instance_variable_get("@#{field}").nil? }
-        end
+        # Job specific attributes and flags
+        Job::FIELDS.each { |name| expose name }
 
         # Technical fields
         expose :wid, unless: lambda { |object, options| object.wid.nil? }

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -63,10 +63,13 @@ module RestFtpDaemon
         optional :label, type: String, desc: "Descriptive label for this job"
         optional :notify, type: String, desc: "URL to get POST'ed notifications back"
         optional :priority, type: Integer, desc: "Priority level of the job (lower is stronger)"
-        optional :overwrite, type: Boolean, desc: "wether to overwrites files at target server",
-          default: false
-        optional :mkdir, type: Boolean, desc: "wether to create missing directories on target server",
-          default: false
+
+        optional :overwrite, type: Boolean, desc: "overwrites files at target server",
+          default: Settings.transfer[:overwrite]
+        optional :mkdir, type: Boolean, desc: "create missing directories on target server",
+          default: Settings.transfer[:mkdir]
+        optional :tempfile, type: Boolean, desc: "upload to a temp file before renaming it to the target filename",
+          default: Settings.transfer[:tempfile]
       end
 
       post '/jobs/' do
@@ -95,7 +98,7 @@ module RestFtpDaemon
           api_error exception
         else
           status 201
-          present job, :with => RestFtpDaemon::API::Entities::JobPresenter
+          present job, :with => RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
         end
       end
 

data/lib/rest-ftp-daemon/constants.rb

@@ -1,11 +1,12 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
-APP_VER = "0.101"
+APP_VER = "0.103.1"
 
 # Some global constants
 IDENT_JOB_LEN = 4
 IDENT_NOTIF_LEN = 4
 IDENT_RANDOM_LEN = 8
+IDENT_TEMPFILE_LEN = 8
 DEFAULT_LOGS_PIPE_LEN = 15
 DEFAULT_LOGS_ID_LEN = 8
 

data/lib/rest-ftp-daemon/job.rb

@@ -6,35 +6,32 @@ require 'timeout'
 module RestFtpDaemon
   class Job
 
-    FIELDS = [:source, :target, :label, :priority, :notify, :overwrite, :mkdir]
+    FIELDS = [:source, :target, :label, :priority, :notify, :overwrite, :mkdir, :tempfile]
 
-    attr_reader :id
     attr_accessor :wid
 
+    attr_reader :id
     attr_reader :error
     attr_reader :status
 
     attr_reader :queued_at
     attr_reader :updated_at
-
     attr_reader :started_at
     attr_reader :finished_at
 
     attr_reader :params
 
-    FIELDS.each do |field|
-      attr_reader field
+    FIELDS.each do |name|
+      attr_reader name
     end
 
+
     def initialize job_id, params={}
       # Call super
       # super()
 
       # Init context
       @id = job_id.to_s
-      FIELDS.each do |field|
-        instance_variable_set("@#{field.to_s}", params[field])
-      end
       @params = {}
       @updated_at = nil
       @started_at = nil
@@ -44,25 +41,30 @@ module RestFtpDaemon
       @wid = nil
 
       # Logger
-      # @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
 
       # Protect with a mutex
       @mutex = Mutex.new
 
+      # Import query params
+      FIELDS.each do |name|
+        instance_variable_set "@#{name.to_s}", params[name]
+      end
+
+      # Set super-default flags
+      flag_default :mkdir, false
+      flag_default :overwrite, false
+      flag_default :tempfile, false
+
       # Flag current job
       @queued_at = Time.now
       @status = :created
 
       # Send first notification
-      #info "Job.initialize/notify"
       info "Job.initialized"
       client_notify "rftpd.queued"
     end
 
-    def close
-    end
-
     def process
       # Update job's status
       @error = nil
@@ -74,31 +76,31 @@ module RestFtpDaemon
         prepare
 
       rescue RestFtpDaemon::JobMissingAttribute => exception
-        return oops "rftpd.started", exception, :job_missing_attribute
+        return oops "rftpd.started", exception, :missing_attribute
 
       # rescue RestFtpDaemon::JobSourceNotFound => exception
       #   return oops "rftpd.started", exception, :job_source_not_found
 
       rescue RestFtpDaemon::JobUnresolvedTokens => exception
-        return oops "rftpd.started", exception, :job_unresolved_tokens
+        return oops "rftpd.started", exception, :unresolved_tokens
 
       rescue RestFtpDaemon::JobTargetUnparseable => exception
-        return oops "rftpd.started", exception, :job_target_unparseable
+        return oops "rftpd.started", exception, :target_unparseable
 
       rescue RestFtpDaemon::JobTargetUnsupported => exception
-        return oops "rftpd.started", exception, :job_target_unsupported
+        return oops "rftpd.started", exception, :target_unsupported
+
+      rescue URI::InvalidURIError => exception
+        return oops "rftpd.started", exception, :target_invalid
 
       rescue RestFtpDaemon::JobAssertionFailed => exception
-        return oops "rftpd.started", exception, :job_assertion_failed
+        return oops "rftpd.started", exception, :assertion_failed
 
       rescue RestFtpDaemon::RestFtpDaemonException => exception
-        return oops "rftpd.started", exception, :job_prepare_failed
-
-      rescue URI::InvalidURIError => exception
-        return oops "rftpd.started", exception, :job_target_invalid
+        return oops "rftpd.started", exception, :prepare_failed
 
       rescue Exception => exception
-        return oops "rftpd.started", exception, :job_prepare_unhandled, true
+        return oops "rftpd.started", exception, :prepare_unhandled, true
 
       else
         # Prepare done !
@@ -113,47 +115,47 @@ module RestFtpDaemon
         @status = :starting
         transfer
 
+      rescue SocketError => exception
+        return oops "rftpd.ended", exception, :conn_socket_error
+
       rescue Errno::EHOSTDOWN => exception
-        return oops "rftpd.ended", exception, :job_host_is_down
+        return oops "rftpd.ended", exception, :conn_host_is_down
 
       rescue Errno::ENOTCONN => exception
-        return oops "rftpd.ended", exception, :job_connexion_failed
+        return oops "rftpd.ended", exception, :conn_failed
 
       rescue Errno::ECONNREFUSED => exception
-        return oops "rftpd.ended", exception, :job_connexion_refused
-
-      rescue SocketError => exception
-        return oops "rftpd.ended", exception, :job_socket_error
+        return oops "rftpd.ended", exception, :conn_refused
 
       rescue Timeout::Error, Errno::ETIMEDOUT => exception
-        return oops "rftpd.ended", exception, :job_timeout
-
-      rescue Net::FTPPermError => exception
-        return oops "rftpd.ended", exception, :job_perm_error
+        return oops "rftpd.ended", exception, :conn_timeout
 
       rescue OpenSSL::SSL::SSLError => exception
-        return oops "rftpd.ended", exception, :job_openssl_error
+        return oops "rftpd.ended", exception, :conn_openssl_error
+
+      rescue Net::FTPPermError => exception
+        return oops "rftpd.ended", exception, :perm_error
 
       rescue Errno::EMFILE => exception
-        return oops "rftpd.ended", exception, :job_too_many_open_files
+        return oops "rftpd.ended", exception, :too_many_open_files
 
       rescue RestFtpDaemon::JobSourceNotFound => exception
-        return oops "rftpd.ended", exception, :job_source_not_found
+        return oops "rftpd.ended", exception, :source_not_found
 
       rescue RestFtpDaemon::JobTargetFileExists => exception
-        return oops "rftpd.ended", exception, :job_target_file_exists
+        return oops "rftpd.ended", exception, :target_file_exists
 
       rescue RestFtpDaemon::JobTargetShouldBeDirectory => exception
-        return oops "rftpd.ended", exception, :job_target_should_be_directory
+        return oops "rftpd.ended", exception, :target_not_directory
 
       rescue RestFtpDaemon::JobAssertionFailed => exception
-        return oops "rftpd.started", exception, :job_assertion_failed
+        return oops "rftpd.started", exception, :assertion_failed
 
       rescue RestFtpDaemon::RestFtpDaemonException => exception
-        return oops "rftpd.ended", exception, :job_transfer_failed
+        return oops "rftpd.ended", exception, :transfer_failed
 
       rescue Exception => exception
-        return oops "rftpd.ended", exception, :job_transfer_unhandled, true
+        return oops "rftpd.ended", exception, :transfer_unhandled, true
 
       else
         # All done !
@@ -187,19 +189,6 @@ module RestFtpDaemon
       (@finished_at - @started_at).round(2)
     end
 
-    def wander time
-      info "Job.wander #{time}"
-      @wander_for = time
-      @wander_started = Time.now
-      sleep time
-      info "Job.wandered ok"
-    end
-
-    def wandering_time
-      return if @wander_started.nil? || @wander_for.nil?
-      @wander_for.to_f - (Time.now - @wander_started)
-    end
-
     def set attribute, value
       @mutex.synchronize do
         @params || {}
@@ -300,10 +289,11 @@ module RestFtpDaemon
       # Scheme-aware config
       ftp_init
 
-      # Connect remote server, login and chdir
-      ftp_connect_and_login
+      # Connect to remote server and login
+      ftp_connect
+      ftp_login
 
-      # Connect remote server, login and chdir
+      # Change to the right path
       path = Helpers.extract_dirname(@target_url.path).to_s
       ftp_chdir_or_buildpath path
 
@@ -344,10 +334,10 @@ module RestFtpDaemon
     def oops signal_name, exception, error_name = nil, include_backtrace = false
       # Log this error
       error_name = exception.class if error_name.nil?
-      info "Job.oops si[#{signal_name}] er[#{error_name.to_s}] ex[#{exception.class}]"
+      info "Job.oops si[#{signal_name}] er[#{error_name.to_s}] ex[#{exception.class}] #{exception.message}"
 
       # Close ftp connexion if open
-      @ftp.close unless @ftp.nil?
+      @ftp.close unless @ftp.welcome.nil?
 
       # Update job's internal status
       @status = :failed
@@ -374,6 +364,17 @@ module RestFtpDaemon
       client_notify signal_name, error_name, notif_status
     end
 
+    def flag_default name, default
+      # build the flag instance var name
+      variable = "@#{name.to_s}"
+
+      # If it's true or false, that's ok
+      return if [true, false].include? instance_variable_get(variable)
+
+      # Otherwise, set it to the default value
+      instance_variable_set variable, default
+    end
+
     def ftp_init
       # Method assertions
       info "Job.ftp_init asserts"
@@ -398,19 +399,20 @@ module RestFtpDaemon
       @ftp.passive = true
     end
 
-    def ftp_connect_and_login
-      #@status = :ftp_connect
-      # connect_timeout_sec = (Settings.transfer.connect_timeout_sec rescue nil) || DEFAULT_CONNECT_TIMEOUT_SEC
-
-      # Method assertions
-      host = @target_url.host
-      info "Job.ftp_connect connect [#{host}]"
+    def ftp_connect
       @status = :ftp_connect
+      host = @target_url.host
+      info "Job.ftp_connect [#{host}]"
       raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
+
       @ftp.connect(host)
+    end
 
+    def ftp_login
       @status = :ftp_login
-      info "Job.ftp_connect login [#{@target_url.user}]"
+      info "Job.ftp_login [#{@target_url.user}]"
+      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.user.nil? || @target_url.password.nil?
+
       @ftp.login @target_url.user, @target_url.password
     end
 
@@ -504,7 +506,6 @@ module RestFtpDaemon
         else
           # won't overwrite then stop here
           info "Job.ftp_transfer failed: target file exists"
-          @ftp.close
           raise RestFtpDaemon::JobTargetFileExists
         end
       end
@@ -513,12 +514,19 @@ module RestFtpDaemon
       update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_UPDATE_EVERY_KB
       notify_after_sec = Settings.transfer.notify_after_sec rescue nil
 
+      # Compute temp target name
+      target_real = target_name
+      if @tempfile
+        target_real = "#{target_name}.#{Helpers.identifier(IDENT_TEMPFILE_LEN)}-temp"
+        info "Job.ftp_transfer target_real [#{target_real}]"
+      end
+
       # Start transfer
       chunk_size = update_every_kb * 1024
       t0 = tstart = Time.now
       notified_at = Time.now
       @status = :uploading
-      @ftp.putbinaryfile(source_match, target_name, chunk_size) do |block|
+      @ftp.putbinaryfile(source_match, target_real, chunk_size) do |block|
         # Update counters
         @transfer_sent += block.bytesize
         set :transfer_sent, @transfer_sent
@@ -544,6 +552,8 @@ module RestFtpDaemon
         t0 = Time.now
 
         # Notify if requested
+        @status = :uploaded
+        info "Job.ftp_transfer uploaded"
         unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
           notif_status = {
             progress: percent1,
@@ -557,6 +567,13 @@ module RestFtpDaemon
 
       end
 
+      # Rename temp file to target_temp
+      if @tempfile
+        @status = :renaming
+        info "Job.ftp_transfer renaming to #{target_name}"
+        @ftp.rename target_real, target_name
+      end
+
       # Compute final bitrate
       set :transfer_bitrate, get_bitrate(@transfer_total, tstart).round(0)
 

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -39,14 +39,14 @@ module RestFtpDaemon
           # Wait for a job to come into the queue
           worker_status wid, :waiting
           job = $queue.pop
-          info "worker [#{wid}] popped [#{job.id}]"
 
           # Do the job
+          info "worker [#{wid}] processing [#{job.id}]"
           worker_status wid, :processing, job.id
           job.wid = wid
           job.process
           info "worker [#{wid}] processed [#{job.id}]"
-          job.close
+          # job.close
           worker_status wid, :done
 
           # Increment total processed jobs count

data/rest-ftp-daemon.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.date = Time.now.strftime("%Y-%m-%d")
   spec.authors = ["Bruno MEDICI"]
   spec.email = "rest-ftp-daemon@bmconseil.com"
-  spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
+  spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
   spec.summary = "RESTful FTP client daemon"
   spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
   spec.licenses = ["MIT"]

data/rest-ftp-daemon.yml.sample

@@ -11,6 +11,11 @@ defaults: &defaults
     update_every_kb: 1024
     notify_after_sec: 10
 
+    mkdir: true
+    tempfile: true
+    overwrite: false
+
+
   conchita:
     timer: 10
     #clean_failed: 3600

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.101'
+  version: 0.103.1
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-24 00:00:00.000000000 Z
+date: 2015-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,7 +178,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This is a pretty simple FTP client daemon, controlled through a RESTfull
+description: This is a pretty simple FTP client daemon, controlled through a RESTful
   API
 email: rest-ftp-daemon@bmconseil.com
 executables: