rest-ftp-daemon 0.30.1 → 0.41

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YThkNmNkZTEwNzM2YzcwM2I2N2NiOWE5OTQwNDI1N2M0NTFjZWZhOA==
+    MDQwNWFlNDE3ZGUwZGY2ZTc1N2FiMGE3OWNkODMxN2VhODdkNDNiMA==
   data.tar.gz: !binary |-
-    ZTllMzMyNGMzNTE1M2U5ZTBjZTc3MWJmYWNiYWJlNzQwM2NiNmMxNQ==
+    MmM2YmRkMDkwMjM2OTM2YjZlOTcyNjU0NmNmODQwNmJhNjI0YzdlNQ==
 SHA512:
   metadata.gz: !binary |-
-    ODQzMzc5MzRiZDZjZDAyM2QxZWQyMThjNTUyZmNlZWJhOGFiNTJiODAyYWJi
-    YjExZmZiNGMzMDM5ZDRjMmRjYTMxMTNhZGU0ZWNkODMxMzVjZTQ3YWY2MTk5
-    MjYxM2MwNDU5YzM2OWU1MGVjZjc5NTgwYWVjOTdmZTEzNjQ5MGM=
+    YzEzZjdmYzEyNWJhZTRkODc0NmY4NjRjNTMxNmZiZWFmMWIwOTg5YmM3YzE5
+    NjIwYTNlMTgzNzhhYTljYTUzNTk2NTk3YzJmOTgzYmNjNzAwY2NiMjFlMDYz
+    MjU3ZmZmNjBhYTA0ZWUzOWJkY2MyNWI1ZmYyM2M5ZDVmYWNiNTA=
   data.tar.gz: !binary |-
-    NTBhZWM0NDM2ODNiY2ZjNTA0YjYxYWE3MTBlYWUwZmM3NjgxOGZhNmNlZDlk
-    ZGEwOWE1ZDM4ZjU1N2FjYTQ2ZTNlMjllOGE5ODlhMTJjNjA5ZGFhYjFlODdj
-    ZDBjM2Y1ZTVmODdiYTk0YjdkMGViMGE1ZDA4ZjQ0NzdhZWUxODE=
+    ZTEyZmVkYjIwYzQ2NjVkZjEyNTUzOGFjY2NkNGJjMzkyZWIzZDQ4ZWJiOGNi
+    OGUxMGY5Nzk4YzUxZWEzMjE5Mjg3NDhkMDI1MzFjMWYyNjYxMDk2NmYzZDY2
+    ODQ0NzIwMWY1NGQwYWZjYTE1NjRiMjFhNGZiYzI1Y2U2YjEwMjI=

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.22)
+    rest-ftp-daemon (0.30.1)
       grape
       json
+      thin (~> 1.6)
 
 GEM
   remote: http://rubygems.org/
@@ -21,10 +22,12 @@ GEM
     builder (3.2.2)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
+    daemons (1.1.9)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     equalizer (0.0.9)
-    grape (0.8.0)
+    eventmachine (1.0.3)
+    grape (0.9.0)
       activesupport
       builder
       hashie (>= 2.1.0)
@@ -34,11 +37,11 @@ GEM
       rack-accept
       rack-mount
       virtus (>= 1.0.0)
-    hashie (3.2.0)
+    hashie (3.3.1)
     i18n (0.6.11)
     ice_nine (0.11.0)
     json (1.8.1)
-    minitest (5.4.0)
+    minitest (5.4.1)
     multi_json (1.10.1)
     multi_xml (0.5.5)
     rack (1.5.2)
@@ -47,6 +50,10 @@ GEM
     rack-mount (0.8.3)
       rack (>= 1.0.0)
     rake (10.3.2)
+    thin (1.6.2)
+      daemons (>= 1.0.9)
+      eventmachine (>= 1.0.0)
+      rack (>= 1.0.0)
     thread_safe (0.3.4)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)

data/README.md

@@ -5,10 +5,32 @@ This is a pretty simple FTP client daemon, controlled through a RESTfull API.
 
 As of today, its main features are :
 
-* Delegate a transfer job, ```PUT```'ing posting simple JSON structure
+* Delegate a transfer job by ``POST```'ing a simple JSON structure
 * Spawn a dedicated thread to handle this job in its own context
-* Report transfer status, progress and errors for each delegated job
+* Report transfer status, progress and errors for each job in realtime
 * Expose JSON status of workers on ```GET /jobs/``` for automated monitoring
+* Parralelize jobs as soon as they arrive
+
+Expected features in a short-time range :
+
+* Handle job queues
+* Handle job priorities
+* Allow change of priorities or other attributes after a job has been started
+* Provide RESTful notifications to the requesting client
+* Offer a basic dashboard directly within the daemon HTTP interface
+* Periodically send an update-notification with transfer status and progress
+* Allow fallback file source when first file path is unavailable (failover)
+* Some refactoring may be needed after thos steps
+* Provide swagger-style API documentation
+* Authenticate API clients
+
+
+
+Documentation TODO
+------------------------------------------------------------------------------------
+overwrite: any non empty value allows overwriting
+todo: queues
+
 
 
 Installation
@@ -19,7 +41,7 @@ This project is available as a rubygem, requires on ruby >= 1.9.3 and rubygems i
 Get and install the gem from rubygems.org:
 
 ```
-# apt-get install ruby2.1 rubygems
+# apt-get install ruby1.9.3 ruby-dev rubygems gcc g++
 gem install rest-ftp-daemon --no-ri --no-rdoc
 ```
 
@@ -41,6 +63,8 @@ For now, daemon logs to ```APP_LOGTO``` defined in ```lib/config.rb```
 Usage examples
 ------------------------------------------------------------------------------------
 
+Requesting notifications is achieved by passing a "notify" key in the request, with a callback URL. This URL will be called at some points, ``POST```'ing a generic JSON structure with progress information.
+
 Start a job to transfer a file named "file.iso" to a local FTP server
 
 ```
@@ -48,17 +72,17 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 '{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
 ```
 
-Start a job to transfer a file named "file.dmg" to a local FTP server
+Start a job to transfer a file, and request notifications ``POST```'ed on "http://requestb.in/1321axg1"
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg"}' "http://localhost:3000/jobs"
+'{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
 ```
 
 Get status of a specific job based on its name
 
 ```
-curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+curl -H "Content-Type: application/json" -X GET -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
 ```
 
 Delete a specific job based on its name

data/bin/rest-ftp-daemon

@@ -1,23 +1,30 @@
 #!/usr/bin/env ruby
 
 # Libs and init
-require 'thin'
+require "thin"
 
 # Initialize some local constants
 APP_ROOT = File.dirname(__FILE__) + '/../'
-APP_NAME = 'rest-ftp-daemon'
+rackup_file = File.expand_path "#{APP_ROOT}/config.ru"
 
-APP_STARTED = Time.now
-APP_DEFAULT_PORT = 3000
-APP_LOGTO = "/tmp/#{APP_NAME}.log"
+# Include config file
+require File.expand_path "#{APP_ROOT}/lib/rest-ftp-daemon/config.rb"
 
 # Prepare thin
-rackup_file = File.expand_path "#{APP_ROOT}/config.ru"
 argv = ARGV
 argv << ["-R", rackup_file] unless ARGV.include?("-R")
-argv << ["-p", APP_DEFAULT_PORT.to_s] unless ARGV.include?("-p")
+argv << ["-l", "/tmp/thin.log"] unless ARGV.include?("-l")
+argv << ["--daemonize"] unless ARGV.include?("--daemonize")
 argv << ["-e", "production"] unless ARGV.include?("-e")
-argv << ["--daemonize"] unless ARGV.include?("-R")
+argv << ["-p", (RestFtpDaemon::PORT).to_s] unless ARGV.include?("-p")
 #argv << ["--stats", "/stats"] unless ARGV.include?("--stats")
 
-Thin::Runner.new(argv.flatten).run!
+# Start Thin with this rackup configuration
+puts "#{RestFtpDaemon::NAME}: using [#{argv.join (' ')}]"
+begin
+  Thin::Runner.new(argv.flatten).run!
+rescue Thin::PidFileNotFound
+  puts "#{RestFtpDaemon::NAME}: not running (Thin::PidFileNotFound)"
+end
+
+

data/config.ru

@@ -2,5 +2,13 @@
 $LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
 require 'rest-ftp-daemon'
 
+# Some extra constants
+APP_STARTED = Time.now
+
+# Create worker pool
+$queue = RestFtpDaemon::JobQueue.new
+$pool = RestFtpDaemon::WorkerPool.new(1)
+
 # Start REST FTP Daemon
-run RestFtpDaemon::API
+#run Rack::Cascade.new [Rack::File.new("/public"), RestFtpDaemon::API::Root]
+run Rack::Cascade.new [RestFtpDaemon::API::Root]

data/lib/rest-ftp-daemon.rb

@@ -2,9 +2,21 @@
 require 'json'
 require 'grape'
 require 'net/ftp'
+require 'net/http'
+require 'securerandom'
+# require 'celluloid/autostart'
 
 # My libs
 require 'rest-ftp-daemon/config'
 require 'rest-ftp-daemon/exceptions'
+require 'rest-ftp-daemon/common'
+require 'rest-ftp-daemon/job_queue'
+require 'rest-ftp-daemon/worker_pool'
+require 'rest-ftp-daemon/logger'
 require 'rest-ftp-daemon/job'
-require 'rest-ftp-daemon/server'
+require 'rest-ftp-daemon/notification'
+require 'rest-ftp-daemon/api/defaults'
+require 'rest-ftp-daemon/api/jobs'
+require 'rest-ftp-daemon/api/root'
+require 'rest-ftp-daemon/www'
+

data/lib/rest-ftp-daemon/api/defaults.rb

@@ -0,0 +1,58 @@
+module RestFtpDaemon
+  module API
+    module Defaults
+      extend ActiveSupport::Concern
+
+      included do
+        #version 'v1', using: :header, vendor: 'ftven'
+        format :json
+        do_not_route_head!
+        do_not_route_options!
+
+        # before do
+        #   header['Access-Control-Allow-Origin'] = '*'
+        #   header['Access-Control-Request-Method'] = '*'
+        # end
+
+        # Handle authentication
+        # http_basic do |username, password|
+        #   User.authenticate!(username, password)
+        # end
+
+        # # global handler for simple not found case
+        # rescue_from ActiveRecord::RecordNotFound do |e|
+        #   error_response(message: e.message, status: 404)
+        # end
+
+        # # global exception handler, used for error notifications
+        # rescue_from :all do |e|
+        #   if Rails.env.development?
+        #     raise e
+        #   else
+        #     Raven.capture_exception(e)
+        #     error_response(message: "Internal server error", status: 500)
+        #   end
+        # end
+
+        # # HTTP header based authentication
+        # before do
+        #   error!('Unauthorized', 401) unless headers['Authorization'] == "some token"
+        # end
+
+        helpers do
+          def api_error exception
+            {
+            :error => exception.class,
+            :errmsg => exception.message,
+            :backtrace => exception.backtrace.first,
+            #:backtrace => exception.backtrace,
+            }
+          end
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -0,0 +1,151 @@
+module RestFtpDaemon
+  module API
+
+    class Jobs < Grape::API
+      include RestFtpDaemon::API::Defaults
+      logger ActiveSupport::Logger.new APP_LOGTO, 'daily'
+
+      params do
+        optional :overwrite, type: Integer, default: false
+      end
+
+      helpers do
+
+        def threads_with_id job_id
+          $threads.list.select do |thread|
+            next unless thread[:job].is_a? Job
+            thread[:job].id == job_id
+          end
+        end
+
+        def job_describe job_id
+          raise RestFtpDaemon::JobNotFound if ($queue.queued_size==0 && $queue.popped_size==0)
+
+          # Find job with this id
+          found = $queue.all.select { |job| job.id == job_id }.first
+          raise RestFtpDaemon::JobNotFound if found.nil?
+          raise RestFtpDaemon::JobNotFound unless found.is_a? Job
+
+          # Return job description
+          found.describe
+        end
+
+        # def job_delete job_id
+        # end
+
+        def job_list
+          $queue.all.map do |item|
+            next unless item.is_a? Job
+            item.describe
+          end
+        end
+
+      end
+
+      desc "Get information about a specific job"
+      params do
+        requires :id, type: Integer, desc: "job id"
+      end
+      get ':id' do
+        info "GET /jobs/#{params[:id]}"
+        begin
+          response = job_describe params[:id].to_i
+        rescue RestFtpDaemon::JobNotFound => exception
+          status 404
+          api_error exception
+        rescue RestFtpDaemonException => exception
+          status 500
+          api_error exception
+        rescue Exception => exception
+          status 501
+          api_error exception
+        else
+          status 200
+          response
+        end
+      end
+
+      # Delete jobs
+      desc "Kill and remove a specific job"
+      delete ':id' do
+       info "DELETE /jobs/#{params[:name]}"
+       status 501
+        # begin
+        #   response = job_delete params[:id].to_i
+        # rescue RestFtpDaemon::JobNotFound => exception
+        #   status 404
+        #   api_error exception
+        # rescue RestFtpDaemonException => exception
+        #   status 500
+        #   api_error exception
+        # rescue Exception => exception
+        #   status 501
+        #   api_error exception
+        # else
+        #   status 200
+        #   response
+        # end
+      end
+
+      # List jobs
+      desc "Get a list of jobs"
+      get do
+        info "GET /jobs"
+        begin
+          response = job_list
+        rescue RestFtpDaemonException => exception
+          status 501
+          api_error exception
+        rescue Exception => exception
+          status 501
+          api_error exception
+        else
+          status 200
+          response
+        end
+      end
+
+
+      # Spawn a new thread for this new job
+      desc "Create a new job"
+      post do
+        info "POST /jobs: #{request.body.read}"
+        begin
+          # Extract params
+          request.body.rewind
+          params = JSON.parse request.body.read
+
+          # Create a new job
+          job_id = $last_worker_id += 1
+          job = Job.new(job_id, params)
+
+          # And psuh it to the queue
+          $queue.push job
+
+          # Later: start it asynchronously
+          #job.future.process
+
+        rescue JSON::ParserError => exception
+          status 406
+          api_error exception
+        rescue RestFtpDaemonException => exception
+          status 412
+          api_error exception
+        rescue Exception => exception
+          status 501
+          api_error exception
+        else
+          status 201
+          job.describe
+        end
+      end
+
+    protected
+
+    def progname
+      "API::Jobs"
+    end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/root.rb

@@ -0,0 +1,79 @@
+module RestFtpDaemon
+  module API
+
+    class Root < Grape::API
+      include RestFtpDaemon::API::Defaults
+      logger ActiveSupport::Logger.new APP_LOGTO, 'daily'
+      #add_swagger_documentation
+
+      mount RestFtpDaemon::API::Jobs => '/jobs'
+      # mount RestFtpDaemon::API::Workers => '/workers'
+
+      helpers do
+        def info message, level = 0
+          Root.logger.add(Logger::INFO, "#{'  '*level} #{message}", "API::Root")
+        end
+
+        def job_list_by_status
+          statuses = {}
+          alljobs = $queue.all.map do |item|
+            next unless item.is_a? Job
+            statuses[item.get_status] ||= 0
+            statuses[item.get_status] +=1
+          end
+
+          statuses
+        end
+      end
+
+      ######################################################################
+      ####### INIT
+      ######################################################################
+      def initialize
+        $last_worker_id = 0
+        super
+      end
+
+
+      ######################################################################
+      ####### API DEFINITION
+      ######################################################################
+
+      # Server global status
+      get '/' do
+        info "GET /"
+        status 200
+        return  {
+          name: RestFtpDaemon::NAME,
+          hostname: `hostname`.chomp,
+          version: RestFtpDaemon::VERSION,
+          started: APP_STARTED,
+          uptime: (Time.now - APP_STARTED).round(1),
+          status: job_list_by_status,
+          queue_size: $queue.all_size,
+          jobs_queued: $queue.queued.collect(&:id),
+          jobs_popped: $queue.popped.collect(&:id),
+          routes: RestFtpDaemon::API::Root::routes,
+          }
+      end
+
+      # Server test
+      get '/debug' do
+        info "GET /debug/"
+        begin
+          raise RestFtpDaemon::DummyException
+        rescue RestFtpDaemon::RestFtpDaemonException => exception
+          status 501
+          api_error exception
+        rescue Exception => exception
+          status 501
+          api_error exception
+        else
+          status 200
+          {}
+        end
+      end
+
+    end
+  end
+end