rest-ftp-daemon 0.302.2 → 0.302.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4068d4abcf6001eb5996416a20da6e307a64f8e8
-  data.tar.gz: bc1043ae08c65fdbb40e495559fbcac62a2bd50f
+  metadata.gz: 404ecae7bf682e2d02aafe69f1a04acfd5792ba6
+  data.tar.gz: c9b5924aa5387ea7423fd9464c695ffc37a3a424
 SHA512:
-  metadata.gz: 61772cd43a690dbff017305680fde1d0bc0bd5a3d801c829e2d94e6a612451bfe12816baef7cad473167b4f814e41880a9c30c12f3690528b484ecb9def573e5
-  data.tar.gz: aa77fd72fdef6a2a25048fbc5996cf47599197bf820f17ae8df9cba88c92b2efed20fa4857eb037e83ce23ad9f6b3797af3e83361461505ef851ccf97860b5a2
+  metadata.gz: 1e88538745df4eca736c1f9b238e6645a6de0fa412b117d741554d7091c9b4d5eeb741bf2be3bc57f9aad5a399b640a163098700c17700b9e40e146cbf231360
+  data.tar.gz: 6e2afd3392eddfbfd6cb3a8422e604ac78971f56a3f3ed3b253c96e0f986c67e44fdc916bdb3defce12095ac839841670dd4157e4aebd7e33dcf3546b06630b1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.302.2)
+    rest-ftp-daemon (0.302.3)
       api-auth
       chamber
       double-bag-ftps
@@ -55,7 +55,7 @@ GEM
     eventmachine (1.2.0.1)
     facter (2.4.6)
       CFPropertyList (~> 2.2.6)
-    ffi (1.9.10)
+    ffi (1.9.13)
     get_process_mem (0.2.1)
     grape (0.16.2)
       activesupport
@@ -93,7 +93,7 @@ GEM
     parser (2.3.1.2)
       ast (~> 2.2)
     powerpack (0.1.1)
-    pry (0.10.3)
+    pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
@@ -106,19 +106,19 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
-    rspec (3.4.0)
-      rspec-core (~> 3.4.0)
-      rspec-expectations (~> 3.4.0)
-      rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.4)
-      rspec-support (~> 3.4.0)
-    rspec-expectations (3.4.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.1)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-mocks (3.4.1)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-support (3.4.1)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
     rubocop (0.32.1)
       astrolabe (~> 1.3)
       parser (>= 2.2.2.5, < 3.0)

data/README.md

@@ -43,25 +43,28 @@ Features
   * metrics about pools, throughtput, and queues output to NewRelic
 
 
-Project status
+Project status and quick installation
 ------------------------------------------------------------------------------------
 
+#### Stability
+
 Though it may need more robust tests, this gem has been used successfully in production for
 a while without any glitches at France Télévisions.
 
-Expected features in a short-time range :
+#### API Documentation
+
+API documentation was [maintained on Apiary](http://docs.restftpdaemon.apiary.io/) and will me migrated to swagger on day or another.
+
+#### Expected features in a short-time range
 
 * Provide swagger-style API documentation
 * Authenticate API clients
 * Allow more transfer protocols (HTTP POST etc)
 * Expose JSON status of workers on `GET /jobs/` for automated monitoring
 
+#### Installation
 
-
-Installation
-------------------------------------------------------------------------------------
-
-With Ruby (version 2.2 or higher) and rubygems properly installed, you only need :
+With Ruby (version 2.3 or higher) and rubygems properly installed, you only need :
 
 ```
 gem install rest-ftp-daemon
@@ -70,9 +73,29 @@ gem install rest-ftp-daemon
 If that is not the case yet, see section [Debian install preparation](#debian-install-preparation).
 
 
-Usage
+Subsystems
+------------------------------------------------------------------------------------
+
+#### Conchita: jobs queues cleanup
+
+Job queue can be set to automatically cleanup after a certain delay. Entries are removed from the queue when they have been idle (updated_at) for more than X seconds, and in any of the following statuses:
+
+- failed (conchita.clean_failed)
+- finished (conchita.clean_finished)
+- queued, (conchita.clean_queued)
+
+Cleanup is done on a regular basis, every (conchita.timer) seconds.
+
+#### Reporter: metrics collection
+
+[TODO]
+
+
+Usage and examples
 ------------------------------------------------------------------------------------
 
+#### Launching rest-ftp-daemon
+
 You must provide a configuration file for the daemon to start, either explicitly using
 option `--config` or implicitly at `/etc/rest-ftp-daemon.yml`. A sample file is provided, issue
 `--help` to get more info.
@@ -89,7 +112,7 @@ The dashboard will provide an overview at `http://localhost:3000/`
 
 If the daemon appears to exit quickly when launched, it may be caused by logfiles that can't be written (check files permissions or owner).
 
-Launcher options :
+#### Launcher options :
 
 | Param   | Short         | Default       | Description                                                 |
 |-------  |-------------- |-------------  |-----------------------------------------------------------  |
@@ -105,9 +128,6 @@ Launcher options :
 | -v      | --version     |               | Show the current version                                    |
 
 
-Usage and examples
-------------------------------------------------------------------------------------
-
 #### Start a job to transfer a file named "file.iso" to a local FTP server
 
 ```
@@ -159,12 +179,6 @@ GET http://localhost:3000/jobs/1
 ```
 
 
-API Documentation
-------------------------------------------------------------------------------------
-
-API documentation is [maintained on Apiary](http://docs.restftpdaemon.apiary.io/)
-
-
 Configuration
 ------------------------------------------------------------------------------------
 
@@ -185,31 +199,80 @@ As a starting point, `rest-ftp-daemon.yml.sample` is an example config file that
 Default administrator credentials are `admin/admin`. Please change the password in this configuration file before starting any kind of production.
 
 
-Metrics
-------------------------------------------------------------------------------------
-
-[FIXME]
-
-
-
-Logging
-------------------------------------------------------------------------------------
-
-The application will log to paths specified in the configuration file, if any.
-Separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages.
-Providing empty values as paths, will simply activate logging to `STDOUT`.
+Here is the contents of the default configuration (oeverride by passing -c local.yml at startup)
+
+```yaml
+daemonize: true
+port: 3000
+user: rftpd
+# group: rftpd
+# host: "myhost"
+allow_reload: false
+
+pools:                      # number of workers decidated to each pool value
+  default: 2
+  urgent: 1
+
+reporter:                   # the subsytem in charge of reporting metrics, mainly to NewRelic
+  debug: false
+  timer: 10                 # report every X seconds
+
+conchita:
+  debug: false
+  timer: 60                 # do the cleaning up every X seconds
+  garbage_collector: true   # force a garbage collector cleanup when cleaning things up
+  clean_failed: 3600        # after X seconds, clean jobs with status="failed"
+  clean_finished: 3600      # //              //              //       finished
+  clean_queued: 86400       # //              //              //       queued
+
+transfer:
+  debug: false
+  mkdir: true               # build directory tree if missing
+  tempfile: true            # transfer to temporary file, rename after sucessful transfer
+  overwrite: false          # overwrite any target file with the same name
+  timeout: 1800             # jobs running for longer than X seconds will be killed
+  notify_after: 5           # wait at least X seconds between HTTP notifications
+
+  debug_ftp: false
+  debug_sftp: false
+
+  retry_on:                 # job error values that will allow a retry
+    - ftp_perm_error
+    - net_temp_error
+    - conn_reset_by_peer
+    - conn_timed_out
+    - conn_refused
+    - sftp_auth_failed
+    - conn_host_is_down
+    - conn_unreachable
+    - conn_failed
+    - conn_openssl_error
+  retry_max: 5              # maximum number of retries before giving up on that job
+  retry_for: 1800           # maximum time window to retry failed jobs
+  retry_after: 10           # delay to wait before tries
+
+newrelic:
+  debug: false
+  # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  # app_name: "rftpd-bigbusiness-dev"     # app_name used for naming app (used as-is if provided)
+  prefix: "rftpd"                         # app prefix to build app_name
+  # platform: "bigbusiness"               # app platform to build app_name
+
+logs:
+  path:     "/var/log/"
+  thin:     "rftpd-environment-thin.log"
+  newrelic: "rftpd-environment-newrelic.log"
+  queue:    "rftpd-environment-core.log"
+  api:      "rftpd-environment-core.log"
+  workers:  "rftpd-environment-core.log"
+  transfer: "rftpd-environment-workers.log"
+  conchita: "rftpd-environment-workers.log"
+  reporter: "rftpd-environment-workers.log"
+  notify:   "rftpd-environment-workers.log"
 
+```
 
-Job cleanup
-------------------------------------------------------------------------------------
 
-Job queue can be set to automatically cleanup after a certain delay. Entries are removed from the queue when they have been idle (updated_at) for more than X seconds, and in any of the following statuses:
-
-- failed (conchita.clean_failed)
-- finished (conchita.clean_finished)
-- queued, (conchita.clean_queued)
-
-Cleanup is done on a regular basis, every (conchita.timer) seconds.
 
 
 TODO for this document
@@ -245,7 +308,9 @@ You may have to install some extra packages for the compilations to complete.
 # rbenv install --list | grep '2.2'
 ```
 
+```
 # curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.2.3
+```
 
 Otherwise, you way have to update ruby-build to include Ruby 2.2 definitions.
 On Debian, 2.2 is not included in Wheezy and appears in Jessie's version of the package.

data/defaults.yml

@@ -5,22 +5,21 @@ user: rftpd
 # host: "myhost"
 allow_reload: false
 
-pools:
+pools:                      # number of workers decidated to each pool value
   default: 2
   urgent: 1
 
-reporter:
-  debug: false
-  timer: 10
+reporter:                   # the subsytem in charge of reporting metrics, mainly to NewRelic
   debug: false
+  timer: 10                 # report every X seconds
 
 conchita:
   debug: false
-  timer: 60
-  garbage_collector: true
-  clean_failed: 3600
-  clean_finished: 3600
-  clean_queued: 86400
+  timer: 60                 # do the cleaning up every X seconds
+  garbage_collector: true   # force a garbage collector cleanup when cleaning things up
+  clean_failed: 3600        # after X seconds, clean jobs with status="failed"
+  clean_finished: 3600      # //              //              //       finished
+  clean_queued: 86400       # //              //              //       queued
 
 transfer:
   debug: false
@@ -33,7 +32,7 @@ transfer:
   debug_ftp: false
   debug_sftp: false
 
-  retry_on:
+  retry_on:                 # job error values that will allow a retry
     - ftp_perm_error
     - net_temp_error
     - conn_reset_by_peer
@@ -44,15 +43,16 @@ transfer:
     - conn_unreachable
     - conn_failed
     - conn_openssl_error
-  retry_max: 5
-  retry_for: 1800
-  retry_after: 10
+  retry_max: 5              # maximum number of retries before giving up on that job
+  retry_for: 1800           # maximum time window to retry failed jobs
+  retry_after: 10           # delay to wait before tries
 
 newrelic:
+  debug: false
   # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-  prefix: "rftpd"                         # app prefix
-  # platform: "bigbusiness"               # app platform name
-  # app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
+  # app_name: "rftpd-bigbusiness-dev"     # app_name used for naming app (used as-is if provided)
+  prefix: "rftpd"                         # app prefix to build app_name
+  # platform: "bigbusiness"               # app platform to build app_name
 
 logs:
   path:     "/var/log/"

data/lib/rest-ftp-daemon/workers/reporter.rb

@@ -52,11 +52,11 @@ module RestFtpDaemon
           metrics_newrelic[name] = value
         end
       end
-      if @config[:debug]
-        log_debug "reported following metrics to NewRelic", metrics_newrelic
-      else
-        log_debug "reported all metrics to NewRelic"
-      end
+
+      # Don't dump metrics unless we're debugging
+      metrics_newrelic = nil unless @config[:debug]
+      newrelic_app_name = ENV["NEW_RELIC_APP_NAME"]
+      log_debug "reported metrics to NewRelic [#{newrelic_app_name}]", metrics_newrelic
     end
 
   end

data/rest-ftp-daemon.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version      = "0.302.2"
+  spec.version      = "0.302.3"
 
   # Project description
   spec.name                   = "rest-ftp-daemon"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.302.2
+  version: 0.302.3
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-01 00:00:00.000000000 Z
+date: 2016-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -382,7 +382,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.6
 signing_key: 
 specification_version: 4
 summary: RESTful FTP client daemon