rest-ftp-daemon 0.230.3 → 0.231.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f72c3669b691a64c1051949e27b3c5bef4674da
-  data.tar.gz: 5fcab534e9f49e300e496fe87d6a38c4f19f678f
+  metadata.gz: 66b0655649ee2a3e30ef38f68e1af2d49f1f734f
+  data.tar.gz: 6eada364e7297144b6d72f1c3b51a3dd109f0566
 SHA512:
-  metadata.gz: 0e67365b6601e4f2ae720277283a5474252097a04fc58b728327dcf755a80c39c50b77fde796dc3810a872dccd50f5b7c4f574c719dac37e102d02de91bde239
-  data.tar.gz: 1a836471155747b9be417dc4196d1dbcee9abce63502cbe47dc9f9075642ed4fc010e1623350184754b475f7af7fd1579b45c3e16f904f1b71b1647e92af2502
+  metadata.gz: a7ce9aad2b8f6bcf1cbc9bfa806c184b67ad79740279e66e73e3c219579fa29c24a4333bbd0b9203fa7d4f67a26bf7b9dd83a4fe8e4bf3ed8e52d68864cc242e
+  data.tar.gz: 460b89ca8c5d7405fefa708c42c1a6c9e2c6c5ad9511d363d6561797d387591b073458f52af8caff1c3546e5b9fde34b8bed00e54a229fa48ae9d9c2a6088d26

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.230.3)
+    rest-ftp-daemon (0.231.0)
       double-bag-ftps
       facter
       get_process_mem
@@ -19,14 +19,14 @@ GEM
   remote: http://rubygems.org/
   specs:
     CFPropertyList (2.2.8)
-    activesupport (4.2.3)
+    activesupport (4.2.4)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.3.8)
-    ast (2.0.0)
+    ast (2.1.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
     axiom-types (0.1.1)
@@ -45,12 +45,12 @@ GEM
       unf (>= 0.0.5, < 1.0.0)
     double-bag-ftps (0.1.2)
     equalizer (0.0.11)
-    eventmachine (1.0.7)
+    eventmachine (1.0.8)
     facter (2.4.4)
       CFPropertyList (~> 2.2.6)
     ffi (1.9.10)
     get_process_mem (0.2.0)
-    grape (0.12.0)
+    grape (0.13.0)
       activesupport
       builder
       hashie (>= 2.1.0)
@@ -60,13 +60,13 @@ GEM
       rack-accept
       rack-mount
       virtus (>= 1.0.0)
-    grape-entity (0.4.5)
+    grape-entity (0.4.8)
       activesupport
       multi_json (>= 1.3.2)
-    haml (4.0.6)
+    haml (4.0.7)
       tilt
     hashie (3.4.2)
-    http (0.8.12)
+    http (0.9.4)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 1.0.1)
@@ -79,13 +79,13 @@ GEM
     ice_nine (0.11.1)
     json (1.8.3)
     method_source (0.8.2)
-    minitest (5.7.0)
+    minitest (5.8.0)
     multi_json (1.11.2)
     multi_xml (0.5.5)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
     net-ssh (2.9.2)
-    newrelic_rpm (3.12.1.298)
+    newrelic_rpm (3.13.0.299)
     parser (2.2.2.6)
       ast (>= 1.1, < 3.0)
     powerpack (0.1.1)

data/bin/rest-ftp-daemon

@@ -20,18 +20,6 @@ end
 puts
 
 
-# Provide default config file information
-DEFAULT_CONFIG_PATH = File.expand_path "/etc/#{APP_NAME}.yml"
-SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/rest-ftp-daemon.yml.sample")
-#SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/#{APP_NAME}.yml.sample")
-TAIL_MESSAGE = <<EOD
-
-A default configuration is available here: #{SAMPLE_CONFIG_FILE}.
-You should copy it to the expected location #{DEFAULT_CONFIG_PATH}:
-
-sudo cp #{SAMPLE_CONFIG_FILE} #{DEFAULT_CONFIG_PATH}
-EOD
-
 # Detect options from ARGV
 options = {}
 parser = OptionParser.new do |opts|
@@ -50,7 +38,7 @@ parser = OptionParser.new do |opts|
   opts.separator ""
   opts.on_tail("-h", "--help", "Show this message")                    do
     puts opts
-    puts TAIL_MESSAGE unless File.exists?(DEFAULT_CONFIG_PATH)
+    puts TAIL_MESSAGE unless File.exist?(DEFAULT_CONFIG_PATH)
     exit
   end
   opts.on_tail("-v", "--version", "Show version (#{APP_VER})")         { puts APP_VER; exit }
@@ -72,7 +60,7 @@ end
 # Load config, and merge options from ARGV into settings
 # FIXME: file configuration detection could reside in settings.rb
 APP_CONF ||= DEFAULT_CONFIG_PATH
-abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exists? APP_CONF
+abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exist? APP_CONF
 begin
   # Import settings
   require File.expand_path("#{app_root}/lib/rest-ftp-daemon/settings")

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -81,15 +81,15 @@ module RestFtpDaemon
         optional :overwrite,
           type: Boolean,
           desc: "Overwrites files at target server",
-          default: Settings.transfer[:overwrite]
+          default: Settings.at(:transfer, :overwrite)
         optional :mkdir,
           type: Boolean,
           desc: "Create missing directories on target server",
-          default: Settings.transfer[:mkdir]
+          default: Settings.at(:transfer, :mkdir)
         optional :tempfile,
           type: Boolean,
           desc: "Upload to a temp file before renaming it to the target filename",
-          default: Settings.transfer[:tempfile]
+          default: Settings.at(:transfer, :tempfile)
       end
 
       post "/" do

data/lib/rest-ftp-daemon/api/root.rb

@@ -73,12 +73,22 @@ module RestFtpDaemon
           status: $queue.counts_by_status,
           workers: $pool.worker_variables,
           jobs_count: $queue.jobs_count,
-          jobs_queued: $queue.queued_ids,
-          config: Helpers.get_censored_config
+          jobs_queued: $queue.queued_ids
           }
       end
 
 
+####### GET /config
+
+      # Server config
+      get "/config" do
+        log_info "GET /config"
+
+        status 200
+        return Helpers.get_censored_config
+      end
+
+
     end
   end
 end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,8 +1,23 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.230.3"
+APP_VER = "0.231.0"
 
+# Provide default config file information
+APP_LIB = File.expand_path File.dirname(__FILE__)
+APP_ROOT = File.expand_path(File.dirname(__FILE__) + "/../../")
+
+DEFAULT_CONFIG_PATH = File.expand_path "/etc/#{APP_NAME}.yml"
+SAMPLE_CONFIG_FILE = File.expand_path(File.join File.dirname(__FILE__), "/../../rest-ftp-daemon.yml.sample")
+
+#SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/#{APP_NAME}.yml.sample")
+TAIL_MESSAGE = <<EOD
+
+A default configuration is available here: #{SAMPLE_CONFIG_FILE}.
+You should copy it to the expected location #{DEFAULT_CONFIG_PATH}:
+
+sudo cp #{SAMPLE_CONFIG_FILE} #{DEFAULT_CONFIG_PATH}
+EOD
 
 # Jobs and workers
 JOB_RANDOM_LEN          = 8
@@ -40,6 +55,10 @@ LOG_FORMAT_MESSAGE      = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-L
 LOG_NEWLINE             = "\n"
 LOG_INDENT              = "\t"
 
+BIND_PORT_TIMEOUT       = 3
+BIND_PORT_LOCALHOST     = '127.0.0.1'
+
+
 
 # Notifications
 NOTIFY_PREFIX           = "rftpd"
@@ -64,6 +83,10 @@ DASHBOARD_WORKER_STYLES = {
   }
 
 
+# API server
+# API_LISTEN_HOST = "0.0.0.0"
+
+
 # Configuration defaults
 DEFAULT_WORKER_TIMEOUT  = 3600
 DEFAULT_FTP_CHUNK       = 512
@@ -75,4 +98,3 @@ DEFAULT_WORKERS         = 1
 APP_STARTED = Time.now
 APP_LIBS = File.dirname(__FILE__)
 
-

data/lib/rest-ftp-daemon/helpers.rb

@@ -43,7 +43,7 @@ module RestFtpDaemon
     def self.extract_filename path
       return unless path.is_a? String
       # match everything that's after a slash at the end of the string
-      m = path.match /\/?([^\/]+)$/
+      m = path.match(/\/?([^\/]+)$/)
       return m[1] unless m.nil?
     end
 
@@ -61,12 +61,10 @@ module RestFtpDaemon
     end
 
     def self.local_port_used? port
-      ip = "127.0.0.1"
-      timeout = 1
       begin
-        Timeout.timeout(timeout) do
+        Timeout.timeout(BIND_PORT_TIMEOUT) do
           begin
-            TCPSocket.new(ip, port).close
+            TCPSocket.new(BIND_PORT_LOCALHOST, port).close
             true
           rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
             false

data/lib/rest-ftp-daemon/job.rb

@@ -59,7 +59,7 @@ module RestFtpDaemon
       flag_default :tempfile, false
 
       # Read source file size and parameters
-      @notify_after_sec = Settings.transfer.notify_after_sec rescue nil
+      @notify_after_sec = Settings.at(:transfer, :notify_after_sec) rescue nil
 
       # Flag current job
       @queued_at = Time.now
@@ -176,7 +176,6 @@ module RestFtpDaemon
         log_info "Job.process notify [ended]"
         client_notify :ended
       end
-
     end
 
     def get attribute
@@ -440,7 +439,6 @@ module RestFtpDaemon
       if @tempfile
         tempname = "#{target.name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
         #log_info "Job.remote_push tempname [#{tempname}]"
-      else
       end
 
       # Remove any existing version if expected, or test its presence
@@ -536,7 +534,7 @@ module RestFtpDaemon
 
     rescue StandardError => ex
       log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
-     end
+    end
 
     def get_bitrate delta_data, delta_time
       return nil if delta_time.nil? || delta_time.zero?

data/lib/rest-ftp-daemon/job_queue.rb

@@ -103,7 +103,7 @@ module RestFtpDaemon
       log_info "find_by_id (#{id}, #{prefixed}) > #{id}"
 
       # Search in jobs queues
-      @jobs.select { |item| item.id == id }.last
+      @jobs.reverse.find { |item| item.id == id }
     end
 
     def push job
@@ -138,7 +138,7 @@ module RestFtpDaemon
 
     def pop non_block=false
       @mutex.synchronize do
-        while true
+        loop do
           if @queue.empty?
             # info "JobQueue.pop: empty"
             raise ThreadError, "queue empty" if non_block
@@ -174,7 +174,7 @@ module RestFtpDaemon
       before = Time.now - maxage.to_i
 
       # Verbose output ?
-      log_info "JobQueue.expire \t[#{status.to_s}] \tbefore \t[#{before}]" if verbose
+      log_info "JobQueue.expire \t[#{status}] \tbefore \t[#{before}]" if verbose
 
       @mutex.synchronize do
         # Delete jobs from the queue when they match status and age limits
@@ -186,12 +186,11 @@ module RestFtpDaemon
           next if job.updated_at > before
 
           # Ok, we have to clean it up ..
-          log_info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
+          log_info "expire [#{status}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
           log_info "#{LOG_INDENT}unqueued" if @queue.delete(job)
 
           true
         end
-
       end
 
     end
@@ -204,7 +203,7 @@ module RestFtpDaemon
 
     def sort_queue!
       @mutex_counters.synchronize do
-        @queue.sort_by! &:weight
+        @queue.sort_by!(&:weight)
       end
     end
 

data/lib/rest-ftp-daemon/logger.rb

@@ -25,8 +25,7 @@ class Logger
     end
 
     # Prepend plain message to output
-    #output.unshift (prefix1 + message.strip)
-    output.unshift (prefix1 + message)
+    output.unshift prefix1 + message
 
     # Send all this to logger
     add context[:level], output

data/lib/rest-ftp-daemon/notification.rb

@@ -85,10 +85,8 @@ module RestFtpDaemon
       # Handle exceptions
       rescue StandardError => ex
         log_error "EXCEPTION: #{ex.inspect}"
-
     end
 
-
     def log_context
       {
       id: @id,

data/lib/rest-ftp-daemon/path.rb

@@ -18,14 +18,14 @@ module RestFtpDaemon
     end
 
     def size
-      File.size full if File.exists? full
+      File.size full if File.exist? full
     end
 
   private
 
     def extract_filename path
       # match everything that's after a slash at the end of the string
-      m = path.match /\/?([^\/]+)$/
+      m = path.match(/\/?([^\/]+)$/)
       return m[1].to_s unless m.nil?
     end
 

data/lib/rest-ftp-daemon/remote.rb

@@ -14,7 +14,7 @@ module RestFtpDaemon
       @url.user ||= "anonymous"
 
       # Annnounce object
-      log_info "Remote.initialize [#{url.to_s}]"
+      log_info "Remote.initialize [#{url}]"
     end
 
     def connect

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -68,15 +68,13 @@ module RestFtpDaemon
     def chdir_or_create directory, mkdir = false
       # Init, extract my parent name and my own name
       log_info "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
-      parent, current = Helpers.extract_parent(directory)
-
-      fulldir = "/#{directory}"
+      parent, _current = Helpers.extract_parent(directory)
 
       # Access this directory
       begin
         @ftp.chdir "/#{directory}"
 
-      rescue Net::FTPPermError => e
+      rescue Net::FTPPermError => _e
         # If not allowed to create path, that's over, we're stuck
         return false unless mkdir
 
@@ -112,7 +110,6 @@ module RestFtpDaemon
         # Update job status after this block transfer
         yield data.bytesize, destination.name
       end
-
     end
 
     def close
@@ -137,7 +134,7 @@ module RestFtpDaemon
       @ftp = DoubleBagFTPS.new
       @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
       @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
-    end
+   end
 
 
   end

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -30,12 +30,11 @@ module RestFtpDaemon
     def present? target
       log_info "RemoteSFTP.present? [#{target.name}]"
       stat = @sftp.stat! target.full
-      size = "?"
 
-      rescue Net::SFTP::StatusException
-        return false
-      else
-        return stat.size
+    rescue Net::SFTP::StatusException
+      return false
+    else
+      return stat.size
     end
 
     # def remove target
@@ -64,14 +63,14 @@ module RestFtpDaemon
     def chdir_or_create directory, mkdir = false
       # Init, extract my parent name and my own name
       log_info "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
-      parent, current = Helpers.extract_parent(directory)
+      parent, _current = Helpers.extract_parent(directory)
 
       # Access this directory
       begin
         # log_info "   chdir [/#{directory}]"
-        handle = @sftp.opendir! "./#{directory}"
+        @sftp.opendir! "./#{directory}"
 
-      rescue Net::SFTP::StatusException => e
+      rescue Net::SFTP::StatusException => _e
         # If not allowed to create path, that's over, we're stuck
         return false unless mkdir
 

data/lib/rest-ftp-daemon/settings.rb

@@ -3,16 +3,10 @@ require "settingslogic"
 # Configuration class
 class Settings < Settingslogic
   # Read configuration
-  namespace (defined?(APP_ENV) ? APP_ENV : "production")
-  source ((File.exists? APP_CONF) ? APP_CONF : Hash.new)
+  namespace defined?(APP_ENV) ? APP_ENV : "production"
+  source File.exist?(APP_CONF) ? APP_CONF : Hash.new
   suppress_errors true
 
-  # Compute my PID filename
-  def pidfile
-    port = self["port"]
-    self["pidfile"] || "/tmp/#{APP_NAME}.port#{port}.pid"
-  end
-
   # Direct access to any depth
   def at *path
     path.reduce(Settings) { |m, key| m && m[key.to_s] }
@@ -23,31 +17,42 @@ class Settings < Settingslogic
     to_hash.to_yaml(indent: 4, useheader: true, useversion: false )
   end
 
-  def init_defaults
-    Settings["host"] ||= `hostname`.chomp.split(".").first
-  end
-
   def newrelic_enabled?
     Settings.at(:newrelic)
   end
 
-  def init_newrelic
-    # Skip if not enabled
-    return ENV["NEWRELIC_AGENT_ENABLED"] = "false" unless Settings.newrelic_enabled?
+  def init_defaults
+    # Init host if missing
+    Settings["host"] ||= `hostname`.chomp.split(".").first
+
+    # Init PID file name if missing
+    Settings["pidfile"] ||= "/tmp/#{APP_NAME}-#{Settings['host']}-#{Settings['port']}.pid"
 
-    # Enable module
-    ENV["NEWRELIC_AGENT_ENABLED"] = "true"
-    ENV["NEW_RELIC_MONITOR_MODE"] = "true"
+    # Init NEWRELIC env
+    if Settings.newrelic_enabled?
+      # Enable module
+      ENV["NEWRELIC_AGENT_ENABLED"] = "true"
+      ENV["NEW_RELIC_MONITOR_MODE"] = "true"
 
-    # License
-    ENV["NEW_RELIC_LICENSE_KEY"] = Settings.at(:newrelic, :licence)
+      # License
+      ENV["NEW_RELIC_LICENSE_KEY"] = Settings.at(:newrelic, :licence)
 
-    # Appname
-    platform = Settings.at(:newrelic, :platform) || Settings["host"]
-    ENV["NEW_RELIC_APP_NAME"] = "#{APP_NICK}-#{platform}-#{APP_ENV}"
+      # Appname
+      platform = Settings.newrelic[:platform] || Settings["host"]
+      Settings.newrelic[:app_name] ||= "#{APP_NICK}-#{platform}-#{Settings.namespace}"
+      ENV["NEW_RELIC_APP_NAME"] = Settings.newrelic[:app_name]
+
+      # Logfile
+      ENV["NEW_RELIC_LOG"] = Settings.at(:logs, :newrelic)
+    else
+      ENV["NEWRELIC_AGENT_ENABLED"] = "false"
+    end
+
+    # That's it!
+  end
 
-    # Logfile
-    ENV["NEW_RELIC_LOG"] = Settings.at(:logs, :newrelic)
+  def overwrite options
+    Settings.merge!(options) if options.is_a? Enumerable
   end
 
 end

data/lib/rest-ftp-daemon/worker_job.rb

@@ -57,7 +57,6 @@ module RestFtpDaemon
     else
       # Clean job status
       job.wid = nil
-
     end
 
 

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -72,7 +72,6 @@ module RestFtpDaemon
 
     rescue StandardError => ex
       log_error "UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
-
     end
 
     def create_worker_thread wid
@@ -97,12 +96,12 @@ module RestFtpDaemon
       end
     end
 
-  protected
 
-    if Settings.newrelic_enabled?
-      add_transaction_tracer :create_conchita_thread,     category: :task
-      add_transaction_tracer :create_worker_thread,       category: :task
-    end
+
+  if Settings.newrelic_enabled?
+    add_transaction_tracer :create_conchita_thread,     category: :task
+    add_transaction_tracer :create_worker_thread,       category: :task
+  end
 
   end
 end

data/rest-ftp-daemon.yml.sample

@@ -21,8 +21,9 @@ defaults: &defaults
     # clean_queued: 86400
 
   newrelic:
-    licence: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-    platform: devel             # nickname used for naming app
+    licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    platform: "bigbusiness"               # app platform name
+    app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
 
   debug:
     ftp: false

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.230.3
+  version: 0.231.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-22 00:00:00.000000000 Z
+date: 2015-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler