rest-ftp-daemon 0.72b → 0.85.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YjI2MmE1NjJjY2NhZWViMGRlMGEzODYwNDFjZDMwYjViM2EwZWNiZQ==
-  data.tar.gz: !binary |-
-    ZDllYjExM2UwNzI1ZTk4OTFmZWRmMDZlZGQyNzU5Y2Q0NjEzNmFkOA==
+SHA1:
+  metadata.gz: 1c6ed0547c0017afce76bb7b5ffc55e5ddb48981
+  data.tar.gz: 3a40535718c17d96e76a2f332d7e71eef4b3e2ce
 SHA512:
-  metadata.gz: !binary |-
-    OTA1MGZiMDY5Yjc0ZmViZTg3ZGYxNTZmYjViYmFiYjcwMzliMGNhYjZjZmJl
-    OWI2MDNlYmY5YjcxMWZmNDQ4YzgzOWM0OTgxMmRlYjg0ZDkyYTY5Yjc5Y2Ix
-    N2M0OTY3ZjNiZWM2MzEzYTg3OTQ0ZWIyMmE0MTIyYzgyNjk5YWQ=
-  data.tar.gz: !binary |-
-    ZTY1YmZmNmQyOWZiMjAzNmZkNTVlNDMwMWZkZTFkOTRiYmRiMjRkN2UwYjU4
-    NmY3N2FiZjJkYWVjNDFjMjAyMDQxMDEyOTMzNmQ3NjAwYTNjNDhhNWQxNjhl
-    YTA1NDJlNTI5YTllNDkxNzM5MWQyY2FiNDFiMGU0YmEyOTEwYzE=
+  metadata.gz: 77d115e07981f5bf41d5ef0240e50c813f2a908c5015aff532538a8c56fa9fdf4e2bbb06cbee60e2a51199923f1e38655adbee68d549f28ecdf8866ab2ab6d2b
+  data.tar.gz: 98e520e5b73394dd7f3cb187673b51650290fcd50fb9c19dd7badc89837814571f35550c08aef9e5c4a7de8534858f35e5b08390ea00f5ed0c29406b90f91662

data/.ruby-version

@@ -0,0 +1 @@
+2.1.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.63)
+    rest-ftp-daemon (0.84)
       double-bag-ftps
       facter
       grape
@@ -10,12 +10,13 @@ PATH
       settingslogic
       sys-cpu
       thin (~> 1.6)
+      timeout
 
 GEM
   remote: http://rubygems.org/
   specs:
     CFPropertyList (2.2.8)
-    activesupport (4.1.6)
+    activesupport (4.1.7)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -34,9 +35,9 @@ GEM
     double-bag-ftps (0.1.2)
     equalizer (0.0.9)
     eventmachine (1.0.3)
-    facter (2.2.0)
+    facter (2.3.0)
       CFPropertyList (~> 2.2.6)
-    ffi (1.9.5)
+    ffi (1.9.6)
     grape (0.9.0)
       activesupport
       builder
@@ -71,6 +72,7 @@ GEM
       rack (~> 1.0)
     thread_safe (0.3.4)
     tilt (2.0.1)
+    timeout (0.0.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     virtus (1.0.3)

data/README.md

@@ -2,9 +2,15 @@ rest-ftp-daemon
 ====================================================================================
 
 
-
 This is a pretty simple FTP client daemon, controlled through a RESTfull API.
 
+API documentation is [maintained on Apiary](http://docs.restftpdaemon.apiary.io/)
+
+
+
+Features
+------------------------------------------------------------------------------------
+
 As of today, its main features are :
 
 * Allow environment-specific configuration in a YAML file
@@ -19,6 +25,10 @@ As of today, its main features are :
 * Allow authentication in FTP target in a standard URI-format
 * Allow configuration-based path templates to abstract local mounts or remote FTPs (endpoint tokens)
 * Remote supported protocols: FTP and FTPs
+* Allow main file transfer protocols: sFTP, FTPs / FTPes
+* Automatically clean-up jobs after a configurable amount of time (failed, finished)
+* Current bitrate on the last blocks chunk updated in the job attributes
+* Global bitrate on the whole file transfer is re-computed after the transfer finishes
 
 Expected features in a short-time range :
 
@@ -29,34 +39,60 @@ Expected features in a short-time range :
 * Provide swagger-style API documentation
 * Authenticate API clients
 * Allow to specify random remote/local source/target
-* Allow more remote protocols: sFTP, FTPs, HTTP POST, etc
+* Allow more transfer protocols (sFTP, HTTP POST etc)
+
 
 
 Installation
 ------------------------------------------------------------------------------------
 
-This project is available as a rubygem, requires on ruby >= 1.9.3 and rubygems installed.
+This project is available as a rubygem, requires Ruby 2.1 and rubygems installed.
 
-Get and install the gem from rubygems.org:
+You may use ```rbenv``` and ```ruby-build``` to get the right Ruby version. If this is your case, ensure that ruby-build definitions are up-to-date and include ruby-2.1.0
 
 ```
-# apt-get install ruby1.9.3 ruby1.9.1-dev ruby-dev rubygems gcc g++
-gem install rest-ftp-daemon --no-ri --no-rdoc
+# apt-get install ruby-build rbenv
+# ruby-build --definitions | grep '2.1'
+
 ```
+Otherwise, you way have to update ruby-build to include Ruby 2.1.0 definitions.
+On Debian, 2.1.0 is not included in Wheezy and appears in Jessie's version of the package.
 
-Start the daemon:
+Use a dedicated user for the daemon, switch to this user and enable rbenv
 
 ```
-rest-ftp-daemon start
+# adduser --disabled-password --gecos "" rftpd
+# su rftpd -l
+# echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
+# echo 'eval "$(rbenv init -)"' >> ~/.bashrc
 ```
 
-Start the daemon on a specific port :
+
+Install the right ruby version and activate it
 
 ```
-rest-ftp-daemon -p4000 start
+# rbenv install 2.1.0
+# rbenv local 2.1.0
+# rbenv rehash
 ```
 
-Check that the daemon is running and providing its status info
+Update RubyGems and install the gem from rubygems.org
+
+```
+# gem update --system
+# gem install rest-ftp-daemon --no-ri --no-rdoc
+# rbenv rehash
+# rest-ftp-daemon start
+```
+
+Finally start the daemon on the standart port, or on a specific port using ```-p```
+
+```
+# rest-ftp-daemon -p 4000 start
+```
+
+Check that the daemon is running and providing its status info.
+If the daemon seems to exit as soon as it's launched, this may be due to logfiles that cannot be written on (check permissions or owner).
 
 ```
 http://localhost:3200/
@@ -64,6 +100,7 @@ http://localhost:3200/
 
 Configuration
 ------------------------------------------------------------------------------------
+
 Most of the configuration options live in a YAML configuration file, containing two main sections:
 
 * the ``defaults`` section should be left as-is and will be used is no other environment-specific value is provided.
@@ -80,6 +117,7 @@ As a starting point, ``rest-ftp-daemon.yml.sample`` is an exemple config file th
 
 Default administrator credentials are admin/admin. Please change the password in this configuration file before starting any kind of production.
 
+
 Logging
 ------------------------------------------------------------------------------------
 
@@ -87,32 +125,25 @@ The application will not log to any file by default, if not specified in its con
 Otherwise separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages. Providing and empty value will simply activate logging to ``STDOUT``.
 
 
-Usage examples
+Job cleanup
 ------------------------------------------------------------------------------------
 
-* Start a job to transfer a file named "file.iso" to a local FTP server
-
-```
-curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
-```
+Job can be cleanup up after a certain amount of time, when they reach on of these status:
 
-Requesting notifications is achieved by passing a "notify" key in the request, with a callback URL.
-This URL will be called at some points, ``POST```'ing a generic JSON structure with progress information.
+- failed, after conchita.clean_failed seconds
+- finished, after conchita.clean_finished seconds
 
+Cleanup is done on a regular basis, every few seconds (conchita.timer)
 
-* Start a job requesting notifications ``POST```'ed on "http://requestb.in/1321axg1"
 
-```
-curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
-```
+Usage examples
+------------------------------------------------------------------------------------
 
-* Start a job with all the above plus a priority
+* Start a job to transfer a file named "file.iso" to a local FTP server
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
+'{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
 ```
 
 * Start a job using endpoint tokens
@@ -137,72 +168,6 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 '{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
 ```
 
-NB: a special token [RANDOM] helps to generate a random filename when needed
-
-* Get status of a specific job based on its ID
-
-```
-curl -H "Content-Type: application/json" -X GET -D /dev/stdout "http://localhost:3000/jobs/3"
-```
-
-
-* Delete a specific job based on its ID
-
-```
-curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/3"
-```
-
-
-Getting status
-------------------------------------------------------------------------------------
-
-* A global JSON status is provided on ``` GET /status ```
-
-* A nice dashboard gives a global view of the daemon, jobs in queue, and system status, exposed on ``` GET /```
-
-* The server exposes its jobs list on ``` GET /jobs ```
-
-```
-http://localhost:3000/jobs
-```
-
-This query will return a job list :
-
-```
-[
-  {
-    "source": "~/file.dmg",
-    "target": "ftp://anonymous@localhost/incoming/dest2.dmg",
-    "worker_name": "bob-92439-1",
-    "created": "2014-08-01 16:53:08 +0200",
-    "id": 16,
-    "runtime": 17.4,
-    "status": "graceful_ending",
-    "source_size": 37109074,
-    "error": 0,
-    "errmsg": "finished",
-    "progress": 100.0,
-    "transferred": 37100000
-  },
-  {
-    source: "[nas]/file.dmg",
-    target: "[ftp2]/dest4.dmg",
-    notify: "http://requestb.in/1321axg1",
-    updated_at: "2014-09-17 22:56:14 +0200",
-    id: 2,
-    started_at: "2014-09-17 22:56:01 +0200",
-    status: "uploading",
-    error: 0,
-    debug_source: "/Users/bruno/file.dmg",
-    debug_target: "#<URI::FTP:0x007ffa9289e650 URL:ftp://uuuuuuuu:yyyyyyyyy@ftp.xxxxxx.fr/subdir/dest4.dmg>",
-    file_size: 32093208,
-    file_progress: 5.6,
-    file_sent: 1800000
-  }
-]
-```
-
-
 About
 ------------------------------------------------------------------------------------
 

data/bin/rest-ftp-daemon

@@ -1,38 +1,110 @@
 #!/usr/bin/env ruby
 
-# Include libs and config file
-require "thin"
-app_root = File.expand_path(File.dirname(__FILE__) + '/../')
-require File.expand_path("#{app_root}/lib/rest-ftp-daemon/config.rb")
-
-# Build rackup context
-settings_daemonize = [nil, true, 1, "1"].include?(Settings.daemonize)
-argv = ARGV
-argv << ["-e", "development"] unless ARGV.include?("-e")
-argv << ["-p", Settings[:port].to_s] unless ARGV.include?("-p") || Settings[:port].nil?
-argv << ["-l", Settings[:logs][:thin]] unless ARGV.include?("-l") unless Settings[:logs][:thin].nil? unless Settings[:logs].nil?
-argv << ["--daemonize"] if settings_daemonize unless (ARGV.include?("-d") || ARGV.include?("--daemonize"))
-argv << ["-R", File.expand_path("#{app_root}/config.ru")] unless ARGV.include?("-R")
+# Try to load libs
+begin
+  require "thin"
+  require 'optparse'
+  require 'socket'
+  require 'timeout'
+rescue LoadError
+  raise "EXITING: some of basic libs were not found: thin, optparse, socket, timeout"
+end
+puts
+
+# Detect environment from options
+begin
+  OptionParser.new do |opts|
+    opts.on("-e", "--environment ENV")   { |env| APP_ENV = env }
+    opts.on("", "--dev")                 { APP_ENV = "development" }
+  end.parse(ARGV)
+rescue OptionParser::InvalidOption => e
+end
+
+# Load remaining libs and config subsystem
+app_root = File.dirname(__FILE__) + '/../'
+[:constants, :helpers, :config].each do |lib|
+  require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}.rb")
+end
+
+# Setup options and import current ARGV
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename $0} [options] start|stop|restart"
+  opts.on("-e", "--environment ENV", "Environment (#{APP_ENV if defined? APP_ENV})")
+  opts.on("--dev", "Force development environment")
+  opts.on("-p", "--port PORT", "use PORT (#{Settings.port})")          { |port| options["port"] = port.to_i }
+  opts.on("-w", "--workers COUNT", "Use COUNT worker threads (#{Settings['workers']})") { |count| options["workers"] = count.to_i }
+  opts.on("-d", "--daemonize", "Run daemonized in the background")     { |bool| options["daemonize"] = bool || true }
+  opts.on("-P", "--pid FILE", "File to store PID")                     { |file| options["pidfile"] = file }
+  opts.on("-u", "--user NAME", "User to run daemon as (use with -g)")  { |user| options["user"] = user }
+  opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
+  opts.on_tail("-h", "--help", "Show this message")                    { puts opts; exit }
+  opts.on_tail('-v', '--version', "Show version")                      { puts Settings['app_ver']; exit }
+end
+begin
+  parser.order!(ARGV)
+  command = ARGV.shift
+  unless ["start", "stop", "restart"].include? command
+    puts parser
+    exit 1
+  end
+rescue OptionParser::InvalidOption => e
+  puts "EXITING: option parser: #{e.message}"
+  exit 1
+end
+Settings.merge!(options)
 
 # Display compiled configuration
-puts "---"
-puts "Version     \t #{Settings.app_ver}"
-puts "Environment \t #{Settings.namespace}"
-puts "Config file \t #{Settings.source}"
-puts "Parameters  \t #{argv.flatten}"
+puts "--- #{APP_NAME} #{APP_VER}"
+puts "Config file  \t #{APP_CONF}"
+puts "PID file     \t #{Settings.pidfile}"
+puts "Namespace    \t #{Settings.namespace}"
+puts "Network port \t #{Settings['port']}"
+puts "Daemonize    \t #{Settings['daemonize']}"
+puts "User:group   \t #{Settings['user']}:#{Settings['group']}"
+puts
 puts Settings.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion => false )
+puts
 
-# Start Thin with this rackup configuration
-puts "---"
+# Validate network configuration
+if ["start", "restart"].include? command
+  if Settings['port'].nil?
+    puts "ABORTING: Network port is missing"
+    exit 1
+  elsif RestFtpDaemon::Helpers.local_port_used?(Settings['port'])
+    puts "ABORTING: Network port #{Settings['port']} is already in use"
+    exit 1
+  end
+end
+
+# Build final ARGV
+argv = []
+argv << ["-e", Settings.namespace]
+argv << ["-p", Settings["port"].to_s] unless Settings["port"].nil?
+argv << ["--pid", Settings.pidfile]
+argv << ["--log", Settings["logs"]["thin"].to_s] unless Settings["logs"]["thin"].nil? if Settings["logs"].is_a? Enumerable
+argv << ["--daemonize"] if [1, true].include? Settings["daemonize"]
+# User / group
+if Settings["user"] && Settings["group"]
+  argv << ["--user", Settings["user"]]
+  argv << ["--group", Settings["group"]]
+end
+argv << command unless command.nil? rescue nil
+puts "--- Thin ARGV"
+puts argv.flatten.join(' ')
 puts
+
+# Start Thin with this rackup configuration
 begin
   Thin::Runner.new(argv.flatten).run!
+  rescue RuntimeError => e
+    puts "FAILED: RuntimeError: #{e.message}"
   rescue Thin::PidFileExist
-    puts "ERROR: daemon was already running (Thin::PidFileExist)"
+    puts "FAILED: daemon was already running (Thin::PidFileExist)"
   rescue Thin::PidFileNotFound
-    puts "ERROR: daemon was not running (Thin::PidFileNotFound)"
+    puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
   rescue SystemExit
-    puts "EXITING: daemon in the background (SystemExit)"
+    # puts "EXITING: daemon in the background (SystemExit)"
   else
-    puts "EXITING: process ending"
+    # puts "FAILED: process ending"
 end

data/config.ru

@@ -1,24 +1,27 @@
 # Load gem files
-APP_LIBS = File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
-$LOAD_PATH.unshift(APP_LIBS) unless $LOAD_PATH.include?(APP_LIBS)
+load_path_libs = File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
+$LOAD_PATH.unshift(load_path_libs) unless $LOAD_PATH.include?(load_path_libs)
 require 'rest-ftp-daemon'
 
 # Create queue and worker pool
 $queue = RestFtpDaemon::JobQueue.new
-$pool = RestFtpDaemon::WorkerPool.new(Settings.workers.to_i)
+$pool = RestFtpDaemon::WorkerPool.new(Settings[:workers] || DEFAULT_WORKERS)
 
-# Rack middleware
-# use Rack::Etag           # Add an ETag
-# use Rack::Reloader,   0
+# Rack reloader
+unless Settings.namespace == "production"
+  use Rack::Reloader, 0
+end
+
+# Rack authent
 unless Settings.adminpwd.nil?
   use Rack::Auth::Basic, "Restricted Area" do |username, password|
     [username, password] == ['admin', Settings.adminpwd]
   end
 end
-#use Rack::Deflator      # Compress
 
 # Serve static assets
-use Rack::Static, :urls => ["/css", "/images"], :root => "#{Settings.app_lib}/static/"
+use Rack::Static, :urls => ["/css", "/images"], :root => "#{APP_LIBS}/static/"
 
 # Launch the main daemon
-run Rack::Cascade.new [RestFtpDaemon::API::Root]
+run RestFtpDaemon::API::Root
+#run Rack::Cascade.new [RestFtpDaemon::API::Root]