rest-ftp-daemon 0.300.3 → 0.302.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 261bf46659b553d4503748d18c088522f198dcbd
-  data.tar.gz: e38b90ee56450265d1967fe95b1992af9ac6e483
+  metadata.gz: 7a1b9e5ed6fac0ec90833a3fbb68442a4a780acf
+  data.tar.gz: c87259dbeb55f4ec01bdd252d7e0141938b58941
 SHA512:
-  metadata.gz: eb73ad80937ab57466428b78fd2af4351e43df7e801fcedbfbf1fcaa0a7c331cdfdf4034946de81bd0498e3e701bcd00817a5f538ddb5140600ac6518996ed86
-  data.tar.gz: 942a05aa3d4f0883e6905e9d3ebb5c860a9c908ba99fb35bc22125cbe101fe2d55e7a4682e13763ae91f81c6dfda54da239cc22a45c0ff1b21345321faeb1f72
+  metadata.gz: 309ccbf778b739bf66c2b2008c659865fe80e6a56400b6713c77f846aa6c121c6ad760fa54c7af2db67ad745708be02337cd10a92ef593ca95580b8ad5f5282b
+  data.tar.gz: 12140ba5f1314819d8a0638af66b91ca4c0aac8752885a40b9a0b387574f3d986ce62e51767a03c4ed9336c1f31f5406626654b3ca4dff24389643f60564232c

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.300.3)
+    rest-ftp-daemon (0.302.0)
+      api-auth
       chamber
       double-bag-ftps
       facter
@@ -12,6 +13,7 @@ PATH
       json
       net-sftp
       newrelic_rpm
+      rest-client (~> 1.8)
       settingslogic
       sys-cpu
       thin (~> 1)
@@ -26,7 +28,7 @@ GEM
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    addressable (2.4.0)
+    api-auth (2.0.0)
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
@@ -71,19 +73,13 @@ GEM
     haml (4.0.7)
       tilt
     hashie (3.4.4)
-    http (2.0.1)
-      addressable (~> 2.3)
-      http-cookie (~> 1.0)
-      http-form_data (~> 1.0.1)
-      http_parser.rb (~> 0.6.0)
     http-cookie (1.0.2)
       domain_name (~> 0.5)
-    http-form_data (1.0.1)
-    http_parser.rb (0.6.0)
     i18n (0.7.0)
     ice_nine (0.11.2)
     json (1.8.3)
     method_source (0.8.2)
+    mime-types (2.99.2)
     minitest (5.9.0)
     multi_json (1.12.1)
     multi_xml (0.5.5)
@@ -92,7 +88,8 @@ GEM
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
     net-ssh (3.2.0)
-    newrelic_rpm (3.15.2.317)
+    netrc (0.11.0)
+    newrelic_rpm (3.16.0.318)
     parser (2.3.1.2)
       ast (~> 2.2)
     powerpack (0.1.1)
@@ -100,11 +97,15 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rack (1.6.4)
+    rack (2.0.1)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rainbow (2.1.0)
     rake (11.2.2)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
@@ -152,7 +153,6 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.6)
-  http
   pry
   rake
   rest-ftp-daemon!

data/README.md

@@ -234,17 +234,21 @@ This project is available as a rubygem, requires Ruby 2.2 and RubyGems installed
 #### Using rbenv and ruby-build
 
 You may use `rbenv` and `ruby-build` to get the right Ruby version. If this is your case, ensure that ruby-build definitions are up-to-date and include the right Ruby version.
+You may have to install some extra packages for the compilations to complete.
 
 ```
+# apt-get install libffi-dev zlib1g-dev bison libreadline-dev
 # git clone https://github.com/rbenv/rbenv.git ~/.rbenv
 # git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
 # echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
 # echo 'eval "$(rbenv init -)"' >> ~/.bashrc
-# ruby-build --definitions | grep '2.2'
+# rbenv install --list | grep '2.2'
 ```
 
-Otherwise, you way have to update ruby-build to include Ruby 2.3.0 definitions.
-On Debian, 2.3.0 is not included in Wheezy and appears in Jessie's version of the package.
+# curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.2.3
+
+Otherwise, you way have to update ruby-build to include Ruby 2.2 definitions.
+On Debian, 2.2 is not included in Wheezy and appears in Jessie's version of the package.
 
 #### Dedicated user
 
@@ -289,6 +293,11 @@ gem install eventmachine -v '1.0.8' -- --with-cppflags=-I/usr/local/opt/openssl/
 bundle install
 ```
 
+* If you get ```uncommon.mk:189: recipe for target 'build-ext' failed``` on Debian, you can try with:
+```
+curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.2.3
+```
+
 
 Contributing
 ------------------------------------------------------------------------------------

data/bin/rest-ftp-daemon

@@ -29,7 +29,6 @@ begin
   Conf.init File.dirname(__FILE__) + "/../"
 
   # Parse options and check compliance
-  thinopts = {}
   parser = OptionParser.new do |opts|
     opts.banner = "Usage: #{File.basename $PROGRAM_NAME} [options] start|stop"
     opts.on("-c", "--config CONFIGFILE")     { |value| cmd_config = File.expand_path(value)}
@@ -45,12 +44,13 @@ begin
     opts.separator ""
     opts.on_tail("-h", "--help", "Show this message")  do
       puts opts
-      puts Conf.gen_config_message unless File.exist?(Conf.gen_config_etc)
+      puts Conf.generate(:config_message) unless File.exist?(Conf.generate(:config_etc))
       exit
     end
     opts.on_tail("-v", "--version", "Show version (#{Conf.app_ver})")         { puts(Conf.app_ver) && exit }
 
-  end.order!(ARGV)
+  end
+  parser.order!(ARGV)
 
   # Load Chamber-based configuration
   Conf.prepare config: cmd_config, logfile: nil
@@ -69,6 +69,15 @@ else
 end
 
 
+# Thin: override with cmdline then defaults
+Conf[:daemonize]    = cmd_daemonize  unless cmd_daemonize.nil?
+Conf[:port]         = cmd_port       unless cmd_port.nil?
+Conf[:user]         = cmd_user       unless cmd_user.nil?
+Conf[:group]        = cmd_group      unless cmd_group.nil?
+Conf[:pidfile]      = cmd_pidfile    unless cmd_pidfile.nil?
+Conf[:pidfile]      ||= Conf.generate(:pidfile)
+
+
 # Validate network configuration
 if command == "start"
   if !Conf[:port]
@@ -79,23 +88,13 @@ if command == "start"
 end
 
 
-# Thin: override with cmdline then defaults
-Conf[:daemonize]    = cmd_daemonize  unless cmd_daemonize.nil?
-Conf[:port]         = cmd_port       unless cmd_port.nil?
-Conf[:user]         = cmd_user       unless cmd_user.nil?
-Conf[:group]        = cmd_group      unless cmd_group.nil?
-Conf[:pidfile]      = cmd_pidfile    unless cmd_pidfile.nil?
-
-Conf[:pidfile]      ||= Conf.gen_pidfile
-
-
 # Thin: basic options
 thin_logfile = Conf.at(:logs, :thin)
 argv = []
 argv << ["-e",          Conf.app_env]
 argv << ["-p",          Conf[:port].to_s]
 argv << ["--pid",       Conf[:pidfile].to_s]
-argv << ["--tag",       "#{Conf.app_name}/#{Conf.app_env}"]
+argv << ["--tag",       Conf.generate(:process_name)]
 argv << ["--daemonize"] if ([1, true].include? Conf[:daemonize])
 argv << ["--log",       thin_logfile] if thin_logfile
 
@@ -120,7 +119,7 @@ puts
 puts "Daemonize       \t #{Conf[:daemonize]}"
 puts "Network port    \t #{Conf[:port]}"
 puts "User:group      \t #{Conf[:user]}:#{Conf[:group]}" if Conf[:user] && Conf[:group]
-puts "Newrelic        \t #{Conf.newrelic_enabled? ? Conf.at(:newrelic, :licence) : "no"}"
+puts "Newrelic        \t #{Conf.newrelic_enabled? ? Conf.at(:newrelic, :licence) : 'no'}"
 puts "PID file        \t #{Conf[:pidfile]}"
 puts
 puts Conf.dump

data/config.ru

@@ -1,5 +1,6 @@
 # Load gem files
-load_path_libs = File.expand_path(File.join(File.dirname(__FILE__), "lib"))
+# load_path_libs = File.expand_path(File.join(File.dirname(__FILE__), "lib"))
+load_path_libs = File.expand_path "lib", File.dirname(__FILE__)
 $LOAD_PATH.unshift(load_path_libs) unless $LOAD_PATH.include?(load_path_libs)
 require "rest-ftp-daemon"
 
@@ -19,9 +20,6 @@ unless Conf[:adminpwd].nil?
   end
 end
 
-# NewRelic profiling
-# GC::Profiler.enable if Conf.newrelic_enabled?
-
 # Serve static assets
 use Rack::Static, urls: ["/css", "/js", "/images"], root: "#{Conf.app_libs}/static/"
 
@@ -31,9 +29,5 @@ unless Conf.app_env == "production"
   # use Rack::MiniProfiler
 end
 
-# Set up encodings
-Encoding.default_internal = "utf-8"
-Encoding.default_external = "utf-8"
-
 # Launch the main daemon
 run RestFtpDaemon::API::Root

data/defaults.yml

@@ -1,61 +1,85 @@
-daemonize: true
-port: 3000
-nick: rftpd
-user: rftpd
-# group: rftpd
-# host: "myhost"
-
-pools:
-  default: 2
-
-transfer:
-  # notify_after_sec: 5       # wait at least X seconds between HTTP notifications
-  # mkdir: true               # build directory tree if missing
-  # tempfile: true            # transfer to temporary file, rename after sucessful transfer
-  # overwrite: false          # overwrite any target file with the same name
-  timeout: 1800             # jobs running for longer than X seconds will be killed
-
-retry:
-  # on_errors:
-  #   - ftp_perm_error
-  #   - net_temp_error
-  #   - conn_reset_by_peer
-  #   - conn_timed_out
-  #   - conn_refused
-  #   - sftp_auth_failed
-  #   - conn_host_is_down
-  #   - conn_unreachable
-  #   - conn_failed
-  #   - conn_openssl_error
-  # max_runs: 5
-  # max_age: 1800
-  # delay: 10
-
-conchita:
-  # timer: 60
-  # garbage_collector: true
-  # clean_failed: 3600
-  # clean_finished: 3600
-  # clean_queued: 86400
-
-newrelic:
-  # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-  prefix: "rftpd"                         # app prefix
-  # platform: "bigbusiness"               # app platform name
-  # app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
-
-debug:
-  ftp: false
-  sftp: false
-  conchita: false
-  reporter: fakse
+defaults: &defaults
+  daemonize: true
+  port: 3000
+  user: rftpd
+  # group: rftpd
+  # host: "myhost"
   allow_reload: false
 
-logs:
-  thin:     "/var/log/rftpd-environment-thin.log"
-  queue:    "/var/log/rftpd-environment-core.log"
-  api:      "/var/log/rftpd-environment-core.log"
-  workers:  "/var/log/rftpd-environment-work.log"
-  jobs:     "/var/log/rftpd-environment-work.log"
-  notify:   "/var/log/rftpd-environment-work.log"
-  newrelic: "/var/log/rftpd-environment-newrelic.log"
+  pools:
+    default: 2
+    urgent: 1
+
+  reporter:
+    debug: false
+    timer: 10
+    debug: false
+
+  conchita:
+    debug: false
+    timer: 60
+    garbage_collector: true
+    clean_failed: 3600
+    clean_finished: 3600
+    clean_queued: 86400
+
+  transfer:
+    debug: false
+    mkdir: true               # build directory tree if missing
+    tempfile: true            # transfer to temporary file, rename after sucessful transfer
+    overwrite: false          # overwrite any target file with the same name
+    timeout: 1800             # jobs running for longer than X seconds will be killed
+    notify_after: 5           # wait at least X seconds between HTTP notifications
+
+    debug_ftp: false
+    debug_sftp: false
+
+    retry_on:
+      - ftp_perm_error
+      - net_temp_error
+      - conn_reset_by_peer
+      - conn_timed_out
+      - conn_refused
+      - sftp_auth_failed
+      - conn_host_is_down
+      - conn_unreachable
+      - conn_failed
+      - conn_openssl_error
+    retry_max: 5
+    retry_for: 1800
+    retry_after: 10
+
+  newrelic:
+    # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    prefix: "rftpd"                         # app prefix
+    # platform: "bigbusiness"               # app platform name
+    # app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
+
+  logs:
+    path:     "/var/log/"
+
+    thin:     "rftpd-environment-thin.log"
+    newrelic: "rftpd-environment-newrelic.log"
+    queue:    "rftpd-environment-core.log"
+    api:      "rftpd-environment-core.log"
+    workers:  "rftpd-environment-core.log"
+
+    transfer: "rftpd-environment-workers.log"
+    conchita: "rftpd-environment-workers.log"
+    reporter: "rftpd-environment-workers.log"
+    notify:   "rftpd-environment-workers.log"
+
+test:
+  <<: *defaults
+
+development:
+  <<: *defaults
+  port: 3100
+
+pre-production:
+  <<: *defaults
+  port: 3200
+
+production:
+  <<: *defaults
+  port: 3300

data/lib/rest-ftp-daemon.rb

@@ -6,11 +6,7 @@ require "haml"
 require "uri"
 require "timeout"
 require "syslog"
-require "net/http"
 require "thread"
-require "singleton"
-require "grape"
-require "grape-entity"
 require "newrelic_rpm"
 
 
@@ -18,6 +14,7 @@ require "newrelic_rpm"
 require_relative "shared/logger_formatter"
 require_relative "shared/logger_helper"
 require_relative "shared/conf"
+require_relative "shared/worker_base"
 
 
 # Project's libs
@@ -31,11 +28,10 @@ require_relative "rest-ftp-daemon/paginate"
 require_relative "rest-ftp-daemon/uri"
 require_relative "rest-ftp-daemon/job_queue"
 require_relative "rest-ftp-daemon/counters"
-require_relative "rest-ftp-daemon/worker"
-require_relative "rest-ftp-daemon/worker_conchita"
-require_relative "rest-ftp-daemon/worker_reporter"
-require_relative "rest-ftp-daemon/worker_job"
 require_relative "rest-ftp-daemon/worker_pool"
+require_relative "rest-ftp-daemon/workers/conchita"
+require_relative "rest-ftp-daemon/workers/reporter"
+require_relative "rest-ftp-daemon/workers/transfer"
 require_relative "rest-ftp-daemon/job"
 require_relative "rest-ftp-daemon/notification"
 

data/lib/rest-ftp-daemon/api/config.rb

@@ -1,3 +1,5 @@
+require "grape"
+
 module RestFtpDaemon
   module API
     class Config < Grape::API
@@ -11,7 +13,7 @@ module RestFtpDaemon
 
       desc "Reload daemon config"
       post "/reload" do
-        if Conf.at(:debug, :allow_reload)==true
+        if Conf[:allow_reload]==true
           Conf.reload!
           status 200
           return Helpers.get_censored_config

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -2,6 +2,7 @@ require "haml"
 require "sys/cpu"
 require "get_process_mem"
 require "facter"
+require "grape"
 
 module RestFtpDaemon
   module API

data/lib/rest-ftp-daemon/api/debug.rb

@@ -1,3 +1,5 @@
+require "grape"
+
 module RestFtpDaemon
   module API
     class Debug < Grape::API

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -1,3 +1,5 @@
+require "grape-entity"
+
 module RestFtpDaemon
   module API
     module Entities