rest-ftp-daemon 0.222.0 → 0.230.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a37b551aba1fb9064471f18ffa0d783ac5b7aab5
-  data.tar.gz: e89ea4d47e9fea2cf75bbabf241b25cb1f3a0c72
+  metadata.gz: 77ab7d0648a235291fd5cd1f3c0682dc5296d8f7
+  data.tar.gz: cfa1db6e26b429e8228b11e65a127774fa270957
 SHA512:
-  metadata.gz: 1295e3ea0b82bad61532d4addca8b05b4f2a5372fa0373e03838b608dab74c26f5439bb09647eee73296746006501427acd4123ea110f13f4cd8c592a3f21c01
-  data.tar.gz: e8a6d8a799496cbf1e51bf6cb77e3424df91f9fe423f5ce9743ceddb9648c9518d7215f3729e8c30c48e150a98da5d99c07415b212215b31ee483f10fe5da049
+  metadata.gz: 10059f2656187713a533efa90dbc3100f6092194d97d11d7e95e4ffa9b1dc0ab6a4147f666496d8cf4acdc190526e7d4c828fc90803b0f1488d931fd02110cbb
+  data.tar.gz: a0d91e6e6d468c1c693e69b4b8fdfc3de62e70e41646543da9d2a5f3cc63f36599a8d7e5cfce151c8efdfb48d8d18221bc70b271b8fdc9cce71c6c991cee76e6

data/.gitignore

@@ -6,3 +6,4 @@ tmp/
 log/
 DOC/
 rest-ftp-daemon.yml
+.ruby-version

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.222.0)
+    rest-ftp-daemon (0.230.0)
       double-bag-ftps
       facter
       get_process_mem
@@ -9,6 +9,7 @@ PATH
       grape-entity
       haml
       json
+      net-sftp
       newrelic_rpm
       settingslogic
       sys-cpu
@@ -19,21 +20,25 @@ GEM
   remote: http://rubygems.org/
   specs:
     CFPropertyList (2.2.8)
-    activesupport (4.2.1)
+    activesupport (4.2.3)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.3.8)
+    ast (2.0.0)
+    astrolabe (1.3.0)
+      parser (>= 2.2.0.pre.3, < 3.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
     builder (3.2.2)
+    coderay (1.1.0)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
-    daemons (1.2.2)
+    daemons (1.2.3)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
@@ -44,9 +49,9 @@ GEM
     eventmachine (1.0.7)
     facter (2.4.4)
       CFPropertyList (~> 2.2.6)
-    ffi (1.9.8)
+    ffi (1.9.10)
     get_process_mem (0.2.0)
-    grape (0.11.0)
+    grape (0.12.0)
       activesupport
       builder
       hashie (>= 2.1.0)
@@ -61,7 +66,7 @@ GEM
       multi_json (>= 1.3.2)
     haml (4.0.6)
       tilt
-    hashie (3.4.1)
+    hashie (3.4.2)
     http (0.8.12)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
@@ -74,30 +79,50 @@ GEM
     i18n (0.7.0)
     ice_nine (0.11.1)
     json (1.8.3)
+    method_source (0.8.2)
     minitest (5.7.0)
-    multi_json (1.11.0)
+    multi_json (1.11.2)
     multi_xml (0.5.5)
-    newrelic_rpm (3.12.0.288)
-    rack (1.6.1)
+    net-sftp (2.1.2)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    newrelic_rpm (3.12.1.298)
+    parser (2.3.0.pre.2)
+      ast (>= 1.1, < 3.0)
+    powerpack (0.1.1)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.4)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-mount (0.8.3)
       rack (>= 1.0.0)
+    rainbow (2.0.0)
     rake (10.4.2)
-    rspec (3.2.0)
-      rspec-core (~> 3.2.0)
-      rspec-expectations (~> 3.2.0)
-      rspec-mocks (~> 3.2.0)
-    rspec-core (3.2.3)
-      rspec-support (~> 3.2.0)
-    rspec-expectations (3.2.1)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.1)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.2.0)
-    rspec-mocks (3.2.1)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.2.0)
-    rspec-support (3.2.2)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    rubocop (0.32.1)
+      astrolabe (~> 1.3)
+      parser (>= 2.2.2.5, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.4)
+    ruby-progressbar (1.7.5)
     settingslogic (2.0.9)
+    slop (3.6.0)
     sys-cpu (0.7.1)
       ffi (>= 1.0.0)
     thin (1.6.3)
@@ -126,6 +151,8 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.6)
   http (~> 0.8)
+  pry
   rake
   rest-ftp-daemon!
   rspec (~> 3.1)
+  rubocop (~> 0.32.0)

data/README.md

@@ -6,11 +6,12 @@ rest-ftp-daemon
 [![Test Coverage](https://codeclimate.com/github/bmedici/rest-ftp-daemon/badges/coverage.svg)](https://codeclimate.com/github/bmedici/rest-ftp-daemon/coverage)
 
 
-A pretty simple FTP-client daemon, driven through a RESTful API.
-
-API documentation is [maintained on Apiary](http://docs.restftpdaemon.apiary.io/)
+A pretty simple but configurable and efficient FTP-client daemon, driven
+through a RESTful API. Create transfer jobs by POSTing a simple JSON structure,
+be notified of their completion, watch their status on a dedicated dashboard.
 
 
+![Dashboard](dashboard.png)
 
 
 Features
@@ -21,11 +22,11 @@ As of today, its main features are :
 * Offer a basic dashboard directly within the daemon HTTP interface
 * Periodically send an update-notification with transfer status and progress
 * Allow environment-specific configuration in a YAML file
-* Delegate a transfer job by ``POST```'ing a simple JSON structure
+* Delegate a transfer job by `POST`'ing a simple JSON structure
 * Spawn a dedicated thread to handle this job in its own context
 * Report transfer status, progress and errors for each job in realtime
-* Expose JSON status of workers on ```GET /jobs/``` for automated monitoring
-* Parralelize jobs as soon as they arrive
+* Expose JSON status of workers on `GET /jobs/` for automated monitoring
+* Parallelize jobs as soon as they arrive
 * Handle job queues and priority as an attribute of the job
 * Allow dynamic evaluation of priorities, and change of any attribute until the job is picked
 * Provide RESTful notifications to the requesting client
@@ -37,8 +38,8 @@ As of today, its main features are :
 * Automatically clean-up jobs after a configurable amount of time (failed, finished)
 * Current bitrate on the last blocks chunk updated in the job attributes
 * Global bitrate on the whole file transfer is re-computed after the transfer finishes
-* Daemon process is tagged with its name and environement in process lists
-* Allow basic patterns in source filename to match multiple files (/dir/file*.jpg)
+* Daemon process is tagged with its name and environment in process lists
+* Allow basic patterns in source filename to match multiple files (`/dir/file*.jpg`)
 
 Expected features in a short-time range :
 
@@ -47,86 +48,42 @@ Expected features in a short-time range :
 * Authenticate API clients
 * Allow more transfer protocols (sFTP, HTTP POST etc)
 
-Known bugs :
+Status
+------------------------------------------------------------------------------------
 
-* As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
+Though lacking testing, this gem has been used successfully in production for
+a while without glitches.
 
 
 Installation
 ------------------------------------------------------------------------------------
 
-This project is available as a rubygem, requires Ruby 2.1 and rubygems installed.
-
-You may use ```rbenv``` and ```ruby-build``` to get the right Ruby version. If this is your case, ensure that ruby-build definitions are up-to-date and include ruby-2.1.0
+With Ruby (version 2.1 or higher) and rubygems properly installed, you only
+need to issue :
 
 ```
-# apt-get install ruby-build rbenv
-# ruby-build --definitions | grep '2.1'
+gem install rest-ftp-daemon
 ```
 
-Otherwise, you way have to update ruby-build to include Ruby 2.1.0 definitions.
-On Debian, 2.1.0 is not included in Wheezy and appears in Jessie's version of the package.
+If that is not the case yet, see section [Debian install preparation](#debian-install-preparation).
 
-Use a dedicated user for the daemon, switch to this user and enable rbenv
-
-```
-# adduser --disabled-password --gecos "" rftpd
-# su rftpd -l
-# echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
-# echo 'eval "$(rbenv init -)"' >> ~/.bashrc
-```
-
-
-Install the right ruby version and activate it
 
-```
-# rbenv install 2.1.0
-# rbenv local 2.1.0
-# rbenv rehash
-```
-
-Update RubyGems and install the gem from rubygems.org
-
-```
-# gem update --system
-# gem install rest-ftp-daemon --no-ri --no-rdoc
-# rbenv rehash
-# rest-ftp-daemon start
-```
-
-
-Configuration
+Usage
 ------------------------------------------------------------------------------------
 
-Most of the configuration options live in a YAML configuration file, containing two main sections:
-
-* the ``defaults`` section should be left as-is and will be used is no other environment-specific value is provided.
-* the ``production`` section can receive personnalized settings according to your environment-specific setup and paths.
-
-Configuration priority is defined as follows (from most important to last resort):
-
-* command-line parameters
-* config file defaults section
-* config file environment section
-* application internal defaults
-
-As a starting point, ``rest-ftp-daemon.yml.sample`` is an exemple config file that can be  copied into the expected location ``/etc/rest-ftp-daemon.yml``.
-
-Default administrator credentials are admin/admin. Please change the password in this configuration file before starting any kind of production.
-
-
-Execution
-------------------------------------------------------------------------------------
+You must provide a configuration file for the daemon to start, either
+explicitly using option `--config` or implicitly at `/etc/rest-ftp-daemon.yml`.
+(A sample file is provided see `--help` for more info about it.)
 
-You can simply start the daemon on the standart port, or on a specific port using ```-p```
+You can then simply start the daemon on the standard port, or on a specific port using `-p`
 
 ```
-# rest-ftp-daemon -p 4000 start
+$ rest-ftp-daemon -p 3000 start
 ```
 
-Check that the daemon is running and exposes a JSON status structure on ```http://localhost:3200/status```.
+Check that the daemon is running and exposes a JSON status structure on `http://localhost:3000/status`.
 
-The dashbaord will provide a gobal view on ```http://localhost:3200/```
+The dashboard will provide a global view on `http://localhost:3000/`
 
 If the daemon appears to exit quickly when launched, it may be caused by logfiles that can't be written (check files permissions or owner).
 
@@ -147,35 +104,17 @@ Launcher options :
 | -v      | --version     |               | Show the current version                                    |
 
 
-Logging
-------------------------------------------------------------------------------------
-
-The application will not log to any file by default, if not specified in its configuration.
-Otherwise separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages. Providing and empty value will simply activate logging to ``STDOUT``.
-
-
-Job cleanup
+Examples
 ------------------------------------------------------------------------------------
 
-Job can be cleanup up after a certain delay, when they are on one of these status:
-
-- "failed", cleaned up after conchita.clean_failed seconds
-- "finished", cleaned up after conchita.clean_finished seconds
-
-Cleanup is done on a regular basis, every X seconds (X = conchita.timer)
-
-
-Usage examples
-------------------------------------------------------------------------------------
-
-* Start a job to transfer a file named "file.iso" to a local FTP server
+#### Start a job to transfer a file named "file.iso" to a local FTP server
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 '{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
 ```
 
-* Start a job using endpoint tokens
+#### Start a job using endpoint tokens
 
 First define ``nas`` ans ``ftp1`` in the configuration file :
 
@@ -190,7 +129,7 @@ development:
     ftp1: "ftp://anonymous@localhost/incoming/"
 ```
 
-Thos tokens will be expanded when the job is ran :
+Those tokens will be expanded when the job is run:
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
@@ -198,16 +137,60 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 ```
 
 
-* Get info about a job with ID="q89j.1"
+#### Get info about a job with ID="q89j.1"
 
-Both parameters ```q89j.1``` and ```1``` will be accepted as ID in the API. Requests below are equivalent:
+Both parameters `q89j.1` and `1` will be accepted as ID in the API. Requests below are equivalent:
 
 ```
-GET http://localhost:3100/jobs/q89j.1
-GET http://localhost:3100/jobs/1
+GET http://localhost:3000/jobs/q89j.1
+GET http://localhost:3000/jobs/1
 ```
 
 
+API Documentation
+------------------------------------------------------------------------------------
+
+API documentation is [maintained on Apiary](http://docs.restftpdaemon.apiary.io/)
+
+
+Configuration
+------------------------------------------------------------------------------------
+
+Most of the configuration options live in a YAML configuration file, containing two main sections:
+
+* `defaults` section should be left as-is and will be used is no other environment-specific value is provided.
+* `production` section can receive personalized settings according to your environment-specific setup and paths.
+
+Configuration priority is defined as follows (from most important to last resort):
+
+* command-line parameters
+* config file defaults section
+* config file environment section
+* application internal defaults
+
+As a starting point, `rest-ftp-daemon.yml.sample` is an example config file that can be  copied into the expected location ``/etc/rest-ftp-daemon.yml``.
+
+Default administrator credentials are `admin/admin`. Please change the password in this configuration file before starting any kind of production.
+
+
+Logging
+------------------------------------------------------------------------------------
+
+The application will not log to any file by default, if not specified in its configuration.
+Otherwise separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages. Providing and empty value will simply activate logging to `STDOUT`.
+
+
+Job cleanup
+------------------------------------------------------------------------------------
+
+Job can be cleanup up after a certain delay, when they are on one of these status:
+
+- "failed", cleaned up after conchita.clean_failed seconds
+- "finished", cleaned up after conchita.clean_finished seconds
+
+Cleanup is done on a regular basis, every X seconds (X = conchita.timer)
+
+
 TODO for this document
 ------------------------------------------------------------------------------------
 
@@ -222,6 +205,89 @@ TODO for this document
 * Document counters
 
 
+
+Debian install preparation
+------------------------------------------------------------------------------------
+
+This project is available as a rubygem, requires Ruby 2.1 and rubygems installed.
+
+You may use `rbenv` and `ruby-build` to get the right Ruby version. If this is your case, ensure that ruby-build definitions are up-to-date and include ruby-2.1.0
+
+```
+# apt-get install ruby-build rbenv
+# ruby-build --definitions | grep '2.1'
+```
+
+Otherwise, you way have to update ruby-build to include Ruby 2.1.0 definitions.
+On Debian, 2.1.0 is not included in Wheezy and appears in Jessie's version of the package.
+
+Use a dedicated user for the daemon, switch to this user and enable rbenv
+
+```
+# adduser --disabled-password --gecos "" rftpd
+# su rftpd -l
+# echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
+# echo 'eval "$(rbenv init -)"' >> ~/.bashrc
+```
+
+
+Install the right ruby version and activate it
+
+```
+# rbenv install 2.1.0
+# rbenv local 2.1.0
+# rbenv rehash
+```
+
+Update RubyGems and install the gem from rubygems.org
+
+```
+# gem update --system
+# gem install rest-ftp-daemon --no-ri --no-rdoc
+# rbenv rehash
+# rest-ftp-daemon start
+```
+
+Known bugs
+------------------------------------------------------------------------------------
+
+* As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
+
+
+Contributing
+------------------------------------------------------------------------------------
+
+Contributions are more than welcome, be it for documentation, features, tests,
+refactoring, you name it. If you are unsure of where to start, the [Code
+Climate](https://codeclimate.com/github/bmedici/rest-ftp-daemon) report will
+provide you with improvement directions. And of course, if in doubt, do not
+hesitate to open an issue. (Please note that this project has adopted a [code
+of conduct](CODE_OF_CONDUCT.md).)
+
+If you want your contribution to adopted in the smoothest and fastest way, don't
+forget to:
+
+* provide sufficient documentation in you commit and pull request
+* add proper testing (we know full grown solid test coverage is still lacking and
+  need to up the game)
+* use the [RuboCop](https://github.com/bbatsov/rubocop) guidelines provided
+  (there are all sorts of editor integration plugins available)
+
+So,
+
+1. Fork the project
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Code
+  * add proper tests if adding a feature
+  * run the tests using `rake`
+  * check for RuboCop style guide violations
+
+4. Commit your changes
+5. Push to the branch (`git push origin my-new-feature`)
+
+6. Create new Pull Request
+
+
 About
 ------------------------------------------------------------------------------------