rest-ftp-daemon 0.250.5 → 0.300.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35b3d7eec8a3e0e3434d6814dd730d48d8797363
-  data.tar.gz: 4d0056bdedf9c43ccb5bf5efc926e29e2bc37cfb
+  metadata.gz: 1cd2717929c1af430db95ad5e9798060d8ab9e2f
+  data.tar.gz: 9fbd387461378d7523194b5bf3e09600661546c4
 SHA512:
-  metadata.gz: 24fd77d7164fd86c3061cf37e7089ed35bf47fba7251cada6e4efaca6b3db1aab7c6d4d052fb323561b7b3fdb8fc1511195c5ee8f4b6d05ae0c69edb8ce2adbe
-  data.tar.gz: 8c9fe1641df446b5d73eb72ffccb7e8077e231eb51f0a74c4d07ceece34cd2d5837b5c98a0f25b444120726719adee0eaa2d95caf451fe33842ab6f4376fd638
+  metadata.gz: 60b92af09c38e6505da2cb1161729be8402f4a6c401e16ac4a0a9ac4f35b017df2d22e2f25d92c068cc0ae611d7a540c1eab189a5ae8f2b56b806ee58f1f1e3d
+  data.tar.gz: ca4061e657d63405810c690aa36f2f98565d26bfe248dcac6389d151a62fb877d8e785722de03e0f8e6714932ccd7b5cde457d7e6757b639f6e36755ec985ac0

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.250.5)
+    rest-ftp-daemon (0.300.1)
+      chamber
       double-bag-ftps
       facter
       get_process_mem
@@ -13,7 +14,7 @@ PATH
       newrelic_rpm
       settingslogic
       sys-cpu
-      thin (~> 1.6)
+      thin (~> 1)
 
 GEM
   remote: http://rubygems.org/
@@ -26,7 +27,7 @@ GEM
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.4.0)
-    ast (2.2.0)
+    ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
     axiom-types (0.1.1)
@@ -34,6 +35,9 @@ GEM
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
     builder (3.2.2)
+    chamber (2.9.0)
+      hashie (~> 3.3)
+      thor (~> 0.19.1)
     coderay (1.1.1)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
@@ -41,7 +45,7 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
-    domain_name (0.5.20160310)
+    domain_name (0.5.20160615)
       unf (>= 0.0.5, < 1.0.0)
     double-bag-ftps (0.1.2)
     enumerable-lazy (0.0.1)
@@ -50,7 +54,7 @@ GEM
     facter (2.4.6)
       CFPropertyList (~> 2.2.6)
     ffi (1.9.10)
-    get_process_mem (0.2.0)
+    get_process_mem (0.2.1)
     grape (0.16.2)
       activesupport
       builder
@@ -67,7 +71,7 @@ GEM
     haml (4.0.7)
       tilt
     hashie (3.4.4)
-    http (0.9.9)
+    http (2.0.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 1.0.1)
@@ -83,13 +87,13 @@ GEM
     minitest (5.9.0)
     multi_json (1.12.1)
     multi_xml (0.5.5)
-    mustermann19 (0.4.3)
+    mustermann19 (0.4.4)
       enumerable-lazy
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
-    net-ssh (3.1.1)
+    net-ssh (3.2.0)
     newrelic_rpm (3.15.2.317)
-    parser (2.3.1.0)
+    parser (2.3.1.2)
       ast (~> 2.2)
     powerpack (0.1.1)
     pry (0.10.3)
@@ -100,7 +104,7 @@ GEM
     rack-accept (0.4.5)
       rack (>= 0.4)
     rainbow (2.1.0)
-    rake (11.1.2)
+    rake (11.2.2)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
@@ -125,12 +129,13 @@ GEM
     slop (3.6.0)
     sys-cpu (0.7.2)
       ffi
-    thin (1.6.4)
+    thin (1.7.0)
       daemons (~> 1.0, >= 1.0.9)
       eventmachine (~> 1.0, >= 1.0.4)
-      rack (~> 1.0)
+      rack (>= 1, < 3)
+    thor (0.19.1)
     thread_safe (0.3.5)
-    tilt (2.0.4)
+    tilt (2.0.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     unf (0.1.4)
@@ -147,7 +152,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.6)
-  http (~> 0.8)
+  http
   pry
   rake
   rest-ftp-daemon!

data/README.md

@@ -40,10 +40,10 @@ Features
 * Realtime status reporting
   * realtime transfer status reporting, with progress and errors
   * periodic update notifications sent along with transfer status and progress to an arbitrary URL (JSON resource POSTed)
+  * metrics about pools, throughtput, and queues output to NewRelic
 
 
-
-Status
+Project status
 ------------------------------------------------------------------------------------
 
 Though it may need more robust tests, this gem has been used successfully in production for
@@ -185,6 +185,13 @@ As a starting point, `rest-ftp-daemon.yml.sample` is an example config file that
 Default administrator credentials are `admin/admin`. Please change the password in this configuration file before starting any kind of production.
 
 
+Metrics
+------------------------------------------------------------------------------------
+
+[FIXME]
+
+
+
 Logging
 ------------------------------------------------------------------------------------
 
@@ -272,7 +279,9 @@ Update RubyGems and install the gem from rubygems.org
 Known bugs
 ------------------------------------------------------------------------------------
 
-* As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
+* As this project is based on the Psyck YAML parser, configuration merge from "defaults" section and environment-specific section are broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults". Please repeat whole sections from "defaults".
+
+* As this project is based on Chamber, and it considers hyphens in filename as namespaces, the global /etc/rest-ftp-daemon.yml config file is not parsed (and thus, ignored). Until this is worked around, please specify a config filename on the commandline.
 
 * If you get ```fatal error: 'openssl/ssl.h' file not found when installing ```eventmachine``` on OSX El Capitan, you can try with:
 ```

data/bin/rest-ftp-daemon

@@ -2,122 +2,128 @@
 
 # Try to load external libs, helpers and constants
 begin
-  require "thin"
+  require "rubygems"
   require "optparse"
+  require "thin"
   require "socket"
-  # require "timeout"
-  require "settingslogic"
-rescue LoadError
-  raise "EXITING: some of basic libs were not found: thin, optparse, socket, settingslogic"
+  require_relative "../lib/rest-ftp-daemon/launcher"
+  require_relative "../lib/shared/conf"
+rescue LoadError => e
+  raise "EXITING: some basic libs were not found (#{e.message})"
 end
-[:constants, :helpers].each do |lib|
-  require_relative "../lib/rest-ftp-daemon/#{lib}"
-end
-puts
+include Shared
+include RestFtpDaemon
 
 
-# Detect options from ARGV
-options = {}
-parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{File.basename $PROGRAM_NAME} [options] start|stop"
-  opts.on("-c", "--config CONFIGFILE")                                  { |config| APP_CONF = config }
-  opts.on("-e", "--environment ENV")                                    { |env| APP_ENV = env }
-  opts.on("",   "--dev")                                                { APP_ENV = "development" }
-  opts.on("-p", "--port PORT", "use PORT")                              { |port| options["port"] = port.to_i }
-  opts.on("-d", "--daemonize", "Run daemonized in the background")      { options["daemonize"] = true }
-  opts.on("-f", "--foreground", "Run in the foreground")                { options["daemonize"] = false }
-  opts.on("-P", "--pid FILE", "File to store PID")                      { |file| options["pidfile"] = file }
-  opts.on("-u", "--user NAME", "User to run daemon as (use with -g)")   { |user| options["user"] = user }
-  opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)") { |group| options["group"] = group }
-
-  opts.separator ""
-  opts.on_tail("-h", "--help", "Show this message")                    do
-    puts opts
-    puts TAIL_MESSAGE unless File.exist?(DEFAULT_CONFIG_PATH)
-    exit
-  end
-  opts.on_tail("-v", "--version", "Show version (#{APP_VER})")         { puts(APP_VER) && exit }
-end
-
-# Parse options and check compliance
+# Handle configuration
 begin
-  parser.order!(ARGV)
+  # Defaults
+  cmd_config = nil
+  cmd_pidfile = nil
+  cmd_port = nil
+  cmd_user = nil
+  cmd_group = nil
+  cmd_daemonize = nil
+
+  # Init Chamber-based configuration from Gemspec
+  Conf.init File.dirname(__FILE__) + "/../"
+
+  # Parse options and check compliance
+  thinopts = {}
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: #{File.basename $PROGRAM_NAME} [options] start|stop"
+    opts.on("-c", "--config CONFIGFILE")     { |value| cmd_config = File.expand_path(value)}
+    opts.on("-e", "--environment ENV")       { |value| Conf.app_env = value }
+    opts.on("",   "--dev")                   { Conf.app_env = "development" }
+    opts.on("-p", "--port PORT", "use PORT")                              { |value| cmd_port = value.to_i }
+    opts.on("-d", "--daemonize", "Run daemonized in the background")      { cmd_daemonize = true }
+    opts.on("-f", "--foreground", "Run in the foreground")                { cmd_daemonize = false }
+    opts.on("-P", "--pid FILE", "File to store PID")                      { |value| cmd_pidfile = value }
+    opts.on("-u", "--user NAME", "User to run daemon as (use with -g)")   { |value| cmd_user = value }
+    opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)") { |value| cmd_group = value }
+
+    opts.separator ""
+    opts.on_tail("-h", "--help", "Show this message")  do
+      puts opts
+      puts Conf.gen_config_message unless File.exist?(Conf.gen_config_etc)
+      exit
+    end
+    opts.on_tail("-v", "--version", "Show version (#{Conf.app_ver})")         { puts(Conf.app_ver) && exit }
+
+  end.order!(ARGV)
+
+  # Load Chamber-based configuration
+  Conf.prepare config: cmd_config, logfile: nil
+
+  # Extract command from command line
   command = ARGV.shift
+
 rescue OptionParser::InvalidOption => e
-  abort "EXITING: option parser: #{e.message}"
+  abort "EXITING: InvalidOption: #{e.message} \n #{e.backtrace.to_yaml}"
+rescue ConfigParseError => e
+  abort "EXITING: ConfigParseError: #{e.message} \n #{e.backtrace.to_yaml}"
+rescue StandardError => e
+  abort "EXITING: StandardError: #{e.message} \n #{e.backtrace.to_yaml}"
 else
   abort parser.to_s unless ["start", "stop"].include? command
 end
 
 
-# Load config, and merge options from ARGV into settings
-# FIXME: file configuration detection could reside in settings.rb
-APP_CONF ||= DEFAULT_CONFIG_PATH
-APP_ENV ||= ENV_PRODUCTION
-abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exist? APP_CONF
-begin
-  # Import settings
-  # require File.expand_path("#{app_root}/lib/rest-ftp-daemon/settings")
-  require_relative "../lib/rest-ftp-daemon/settings"
-
-  # Set defaults
-  Settings.init_defaults
-
-  # Overwrite with commandline options
-  Settings.merge!(options)
-
-  # Init NewRelic
-  Settings.init_newrelic
-
-rescue Psych::SyntaxError => e
-  abort "EXITING: config file syntax error: #{e.message}"
-rescue StandardError => e
-  abort "EXITING: unknow error loading settings  #{e.inspect}"
-end
-
-
 # Validate network configuration
 if command == "start"
-  if !Settings.port
+  if !Conf[:port]
     abort "ABORTING: Network port is missing"
-  elsif RestFtpDaemon::Helpers.local_port_used? Settings.port
-    abort "ABORTING: Network port #{Settings.port} is already in use"
+  elsif Launcher.local_port_used? Conf[:port]
+    abort "ABORTING: Network port #{Conf[:port]} is already in use"
   end
 end
 
 
-# ARGV: build final
-argv = []
-argv << ["-e", Settings.namespace]
-argv << ["-p", Settings.port.to_s] if Settings.port
-argv << ["--pid", Settings.pidfile]
-argv << ["--tag", "'#{APP_NAME}/#{Settings.namespace}'"]
-argv << ["--daemonize"] if [1, true].include? Settings.daemonize
+# Thin: override with cmdline then defaults
+Conf[:daemonize]    = cmd_daemonize  unless cmd_daemonize.nil?
+Conf[:port]         = cmd_port       unless cmd_port.nil?
+Conf[:user]         = cmd_user       unless cmd_user.nil?
+Conf[:group]        = cmd_group      unless cmd_group.nil?
+Conf[:pidfile]      = cmd_pidfile    unless cmd_pidfile.nil?
 
+Conf[:pidfile]      ||= Conf.gen_pidfile
 
-# ARGV: logging, user and group, command
-if (logfile = Settings.at :logs, :thin)
-  argv << ["--log", logfile] if logfile
-end
-if Settings.user && Settings.group
-  argv << ["--user", Settings.user]
-  argv << ["--group", Settings.group]
+
+# Thin: basic options
+thin_logfile = Conf.at(:logs, :thin)
+argv = []
+argv << ["-e",          Conf.app_env]
+argv << ["-p",          Conf[:port].to_s]
+argv << ["--pid",       Conf[:pidfile].to_s]
+argv << ["--tag",       "#{Conf.app_name}/#{Conf.app_env}"]
+argv << ["--daemonize"] if ([1, true].include? Conf[:daemonize])
+argv << ["--log",       thin_logfile] if thin_logfile
+
+# Thin: user and group
+if Conf[:user] && Conf[:group]
+  argv << ["--user", Conf[:user].to_s]
+  argv << ["--group", Conf[:group].to_s]
 end
+
+# Thin: command
 argv << command unless command.nil?
 
 
-# Display final configuration
-puts "--- #{APP_NAME} #{APP_VER}"
-puts "Config file  \t #{APP_CONF}"
-puts "PID file     \t #{Settings.pidfile}"
-puts "Daemonize    \t #{Settings.daemonize ? "YES" : "no (PID: #{Process.pid})"}"
-puts "Host         \t #{Settings.host}"
-puts "Namespace    \t #{Settings.namespace}"
-puts "Network port \t #{Settings.port}"
-puts "User:group   \t #{Settings.user}:#{Settings.group}" if Settings.user || Settings.group
-puts "Newrelic     \t #{Settings.newrelic_enabled? ? Settings.at(:newrelic, :licence) : "no"}"
-puts "Started at   \t #{APP_STARTED}"
-# puts Settings.dump
+# Display final configuration, quit if config dump requested
+puts "--- #{Conf.app_name} #{Conf.app_ver}"
+puts "Environment     \t #{Conf.app_env}"
+puts "Config files    \t #{Conf.files}"
+puts "Started at      \t #{Conf.app_started}"
+puts "Loging to file  \t #{Conf[:log][:file]}" if Conf[:log].is_a? Enumerable
+puts "Host            \t #{Conf.host}"
+puts
+puts "Daemonize       \t #{Conf[:daemonize]}"
+puts "Network port    \t #{Conf[:port]}"
+puts "User:group      \t #{Conf[:user]}:#{Conf[:group]}" if Conf[:user] && Conf[:group]
+puts "Newrelic        \t #{Conf.newrelic_enabled? ? Conf.at(:newrelic, :licence) : "no"}"
+puts "PID file        \t #{Conf[:pidfile]}"
+puts
+puts Conf.dump
 puts
 puts "--- Thin ARGV"
 puts argv.flatten.join(" ")
@@ -126,8 +132,7 @@ puts
 
 # Start Thin with this rackup configuration, changing to app_root first
 begin
-  app_root = File.expand_path(File.dirname(__FILE__) + "/../")
-  Dir.chdir app_root
+  Dir.chdir Conf.app_root
   Thin::Runner.new(argv.flatten).run!
 rescue RuntimeError => e
   puts "FAILED: RuntimeError: #{e.message}"
@@ -137,18 +142,19 @@ rescue Thin::PidFileNotFound
   puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
 rescue Errno::ENOENT => e
   puts "FAILED: daemon cannot access files: #{e.message}"
+rescue ArgumentError => e
+  puts "FAILED: argument error: #{e.message}"
 rescue SystemExit
   # Leave some time for the daemon to launch
-  pidfile = Settings.pidfile
-  print "Waiting for pidfile"
-  until File.file?(pidfile) do
+  print "Waiting for pidfile "
+  until File.file?(Conf[:pidfile]) do
     print "."
     sleep 0.25
   end
   puts
 
   # Check that this process is running
-  pid = File.read(pidfile).to_i
+  pid = File.read(Conf[:pidfile]).to_i
   begin
     Process.kill(0, pid)
     puts "Process ##{pid} is running"

data/config.ru

@@ -13,20 +13,20 @@ $counters = RestFtpDaemon::Counters.new
 $pool = RestFtpDaemon::WorkerPool.new
 
 # Rack authent
-unless Settings.adminpwd.nil?
+unless Conf[:adminpwd].nil?
   use Rack::Auth::Basic, "Restricted Area" do |username, password|
-    [username, password] == ["admin", Settings.adminpwd]
+    [username, password] == ["admin", Conf[:adminpwd]]
   end
 end
 
 # NewRelic profiling
-GC::Profiler.enable if Settings.newrelic_enabled?
+# GC::Profiler.enable if Conf.newrelic_enabled?
 
 # Serve static assets
-use Rack::Static, urls: ["/css", "/js", "/images"], root: "#{APP_LIBS}/static/"
+use Rack::Static, urls: ["/css", "/js", "/images"], root: "#{Conf.app_libs}/static/"
 
 # Rack reloader and mini-profiler
-unless Settings.namespace == ENV_PRODUCTION
+unless Conf.app_env == "production"
   # use Rack::Reloader, 1
   # use Rack::MiniProfiler
 end

data/defaults.yml

@@ -0,0 +1,61 @@
+daemonize: true
+port: 3000
+nick: rftpd
+user: rftpd
+# group: rftpd
+# host: "myhost"
+
+pools:
+  default: 2
+
+transfer:
+  # notify_after_sec: 5       # wait at least X seconds between HTTP notifications
+  # mkdir: true               # build directory tree if missing
+  # tempfile: true            # transfer to temporary file, rename after sucessful transfer
+  # overwrite: false          # overwrite any target file with the same name
+  timeout: 1800             # jobs running for longer than X seconds will be killed
+
+retry:
+  # on_errors:
+  #   - ftp_perm_error
+  #   - net_temp_error
+  #   - conn_reset_by_peer
+  #   - conn_timed_out
+  #   - conn_refused
+  #   - sftp_auth_failed
+  #   - conn_host_is_down
+  #   - conn_unreachable
+  #   - conn_failed
+  #   - conn_openssl_error
+  # max_runs: 5
+  # max_age: 1800
+  # delay: 10
+
+conchita:
+  # timer: 60
+  # garbage_collector: true
+  # clean_failed: 3600
+  # clean_finished: 3600
+  # clean_queued: 86400
+
+newrelic:
+  # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  prefix: "rftpd"                         # app prefix
+  # platform: "bigbusiness"               # app platform name
+  # app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
+
+debug:
+  ftp: false
+  sftp: false
+  conchita: false
+  reporter: fakse
+  allow_reload: false
+
+logs:
+  thin:     "/var/log/rftpd-environment-thin.log"
+  queue:    "/var/log/rftpd-environment-core.log"
+  api:      "/var/log/rftpd-environment-core.log"
+  workers:  "/var/log/rftpd-environment-work.log"
+  jobs:     "/var/log/rftpd-environment-work.log"
+  notify:   "/var/log/rftpd-environment-work.log"
+  newrelic: "/var/log/rftpd-environment-newrelic.log"