rest-ftp-daemon 0.6.0 → 0.9.0

data/Gemfile.lock

@@ -13,7 +13,7 @@ GEM
       thread_safe (~> 0.3, >= 0.3.1)
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
-    git (1.2.7)
+    git (1.2.8)
     github_api (0.12.0)
       addressable (~> 2.3)
       descendants_tracker (~> 0.0.4)
@@ -53,7 +53,7 @@ GEM
     rack-protection (1.5.3)
       rack
     rake (10.3.2)
-    rdoc (3.12.2)
+    rdoc (4.1.1)
       json (~> 1.4)
     shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)

data/README.md

@@ -1,90 +1,120 @@
-## rest-ftp-daemon ##
+rest-ftp-daemon
+====================================================================================
+
+This is a pretty simple FTP client daemon, controlled through a RESTfull API.
 
-This is a fairly basic FTP client daemon, driven by REST-based webservice calls.
 As of today, its main features are :
 
-* Start a job through a basic structure posted with PUT /jobs
-* Spawn a dedicated thread to handle this job in its own contexte
-* Report transfer progress, error and activity for each job
-* Gather jobs status into the main process to get a global view
-* Report JSON status of workers on GET /jobs/ for automated monitoring
+* Delegate a transfer job, ```PUT```'ing posting simple JSON structure
+* Spawn a dedicated thread to handle this job in its own context
+* Report transfer status, progress and errors for each delegated job
+* Expose JSON status of workers on ```GET /jobs/``` for automated monitoring
 
 
-## Quick setup ##
+Installation
+------------------------------------------------------------------------------------
 
-This project requires ruby >= 1.9 and rubygems installed.
+This project is available as a rubygem, requires on ruby >= 1.9 and rubygems installed.
 
-Quickly install the gem from rubygems.org:
+Get and install the gem from rubygems.org:
 
- ``` gem install rest-ftp-daemon ```
+```
+gem install rest-ftp-daemon
+```
 
 Start the daemon:
 
-``` rest-ftp-daemon start ```
+```
+rest-ftp-daemon start
+```
+
+Check that the daemon is running and giving status info
+
+```
+http://localhost:3000/
+```
 
-For now, daemon logs to APP_LOGTO defined in lib/config.rb
+For now, daemon logs to ```APP_LOGTO``` defined in ```lib/config.rb```
 
 
-## Basic usage ##
+Usage examples
+------------------------------------------------------------------------------------
 
-Starting a job transferring file named "file.ova"
+Start a job to transfer a file named "file.ova" to a local FTP server
 
-  curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-  '{"source":"~/file.ova","target":"ftp://anonymous@localhost/incoming/dest2.ova"}' "http://localhost:3000/jobs"
+```
+curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+'{"source":"~/file.ova","target":"ftp://anonymous@localhost/incoming/dest2.ova"}' "http://localhost:3000/jobs"
+```
 
+Start a job to transfer a file named "file.dmg" to a local FTP server
 
-Starting a job transferring file named "dmg"
+```
+curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+'{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg"}' "http://localhost:3000/jobs"
+```
 
-  curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-  '{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg"}' "http://localhost:3000/jobs"
+Get status of a specific job based on its name
 
+```
+curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+```
 
-Delete a specific job
+Delete a specific job based on its name
 
-  curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+```
+curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+```
 
 
+Getting status
+------------------------------------------------------------------------------------
 
-## Getting status ##
+The server exposes jobs list on ``` GET /jobs ```
 
-  GET /jobs
+```
+http://localhost:3000/jobs
+```
 
-Would return:
+This query will return a job list :
 
-  [
-    {
-      "id": "bob-49126-8",
-      "process": "sleep",
-      "status": "transferring",
-      "context": {
-        "source": "~\/file.ova",
-        "target": "ftp:\/\/anonymous@localhost\/incoming\/dest2.ova",
-        "code": -1,
-        "errmsg": "running",
-        "source_size": 1849036800,
-        "progress": 1.9,
-        "transferred": 34800000
-      }
-    },
-    {
-      "id": "bob-49126-9",
-      "process": "sleep",
-      "status": "transferring",
-      "context": {
-        "source": "~\/file.dmg",
-        "target": "ftp:\/\/anonymous@localhost\/incoming\/dest4.dmg",
-        "code": -1,
-        "errmsg": "running",
-        "source_size": 37109074,
-        "progress": 32.9,
-        "transferred": 12200000
-      }
-    }
-  ]
+```
+[
+  {
+    "source": "~/file.dmg",
+    "target": "ftp://anonymous@localhost/incoming/dest2.dmg",
+    "worker_name": "bob-92439-1",
+    "created": "2014-08-01 16:53:08 +0200",
+    "id": 16,
+    "runtime": 17.4,
+    "status": "graceful_ending",
+    "source_size": 37109074,
+    "error": 0,
+    "errmsg": "finished",
+    "progress": 100.0,
+    "transferred": 37100000
+  },
+  {
+    "source": "~/file.ova",
+    "target": "ftp://anonymous@localhost/incoming/dest2.ova",
+    "worker_name": "bob-92439-2",
+    "created": "2014-08-01 16:53:12 +0200",
+    "id": 17,
+    "runtime": 13.8,
+    "status": "uploading",
+    "source_size": 1849036800,
+    "error": -1,
+    "errmsg": "uploading",
+    "progress": 36.1,
+    "transferred": 668300000
+  }
+]
+```
 
 
-### About ###
+About
+------------------------------------------------------------------------------------
 
-Bruno MEDICI
+Bruno MEDICI Consultant
 
 http://bmconseil.com/

data/Rakefile

@@ -18,7 +18,7 @@ Jeweler::Tasks.new do |gem|
   gem.homepage = "http://github.com/bmedici/rest-ftp-daemon"
   gem.license = "MIT"
   gem.summary = "RESTful FTP client daemon"
-  gem.description = "A fairly basic FTP client daemon, driven by RESTful webservice calls"
+  gem.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
   gem.email = "rest-ftp-daemon@bmconseil.com"
   gem.authors = ["Bruno"]
   # dependencies defined in Gemfile

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.9.0

data/bin/rest-ftp-daemon

@@ -5,9 +5,11 @@ require 'thin'
 
 # Initialize some local constants
 APP_ROOT = File.dirname(__FILE__) + '/../'
+APP_NAME = 'rest-ftp-daemon'
 APP_VER = File.read "#{APP_ROOT}/VERSION"
 APP_STARTED = Time.now
 APP_DEFAULT_PORT = 3000
+APP_LOGTO = "/tmp/#{APP_NAME}.log"
 
 # Prepare thin
 rackup_file = File.expand_path "#{APP_ROOT}/lib/config.ru"
@@ -15,7 +17,6 @@ argv = ARGV
 argv << ["-R", rackup_file] unless ARGV.include?("-R")
 argv << ["-p", APP_DEFAULT_PORT.to_s] unless ARGV.include?("-p")
 argv << ["-e", "production"] unless ARGV.include?("-e")
-argv << ["-d"] unless ARGV.include?("-d")
+argv << ["--stats", "/stats"] unless ARGV.include?("--stats")
 
-# Start thin
 Thin::Runner.new(argv.flatten).run!

data/lib/config.rb

@@ -1,7 +1,3 @@
-# App configuration
-APP_NAME = 'rest-ftp-daemon'
-APP_LOGTO = "/tmp/#{APP_NAME}.log"
-
 # Transfer configuration
 TRANSFER_CHUNK_SIZE = 100000
 THREAD_SLEEP_BEFORE_DIE = 120

data/lib/errors.rb

@@ -1,8 +1,3 @@
-# Error codes
-# O: all ok
-# 1x: request body errors
-# 2x: transfer errors
-
 ERR_OK                            = 0
 ERR_BUSY                          = -1
 
@@ -13,4 +8,5 @@ ERR_REQ_TARGET_SCHEME             = 13
 ERR_JOB_SOURCE_NOTFOUND            = 21
 ERR_JOB_TARGET_UNPARSEABLE         = 22
 ERR_JOB_PERMISSION                 = 24
+ERR_JOB_TARGET_PRESENT             = 25
 

data/lib/extend_threads.rb

@@ -0,0 +1,14 @@
+class Thread
+
+  def job
+    # Basic fields
+    job = self[:job]
+
+    # Calculated fields
+    job[:id] = self[:id]
+    job[:runtime] = (Time.now - job[:created]).round(1)
+
+    return job
+  end
+
+end

data/lib/rest-ftp-daemon.rb

@@ -4,18 +4,22 @@ class RestFtpDaemon < Sinatra::Base
   configure :development, :production do
 
     # Create new thread group
-    @@threads = ThreadGroup.new
+    @@workers = ThreadGroup.new
+
+    # Logging configuration
+    #use Rack::CommonLogger, logger
 
     # Some other configuration
-    set :sessions, false
-    # set :logging, true
+    disable :sessions
+    disable :logging
   end
 
   # Server initialization
   def initialize
     # Setup logger
     @logger = Logger.new(APP_LOGTO, 'daily')
-    @logger.level = Logger::INFO
+    #@logger = Logger.new
+    #@logger.level = Logger::INFO
 
     # Other stuff
     @@last_worker_id = 0
@@ -42,23 +46,34 @@ class RestFtpDaemon < Sinatra::Base
     # Build response
     content_type :json
     JSON.pretty_generate get_jobs
-    #@@threads.count
   end
 
-  # List jobs
-  delete "/jobs/:name" do
+  # Get job info
+  get "/jobs/:id" do
     # Debug query
-    info "DELETE /jobs/#{params[:name]}"
+    info "GET /jobs/#{params[:id]}"
 
-    # Kill this job
-    ret = delete_job params[:name]
+    # Find this process by name
+    found = find_job params[:id]
 
-    # Fail if no process has been killed
-    error 404 if ret<1
+    # Build response
+    error 404 and return if found.nil?
+    content_type :json
+    JSON.pretty_generate found
+  end
+
+  # Delete jobs
+  delete "/jobs/:id" do
+    # Debug query
+    info "DELETE /jobs/#{params[:name]}"
+
+    # Find and kill this job
+    found = delete_job params[:id]
 
     # Build response
+    error 404 and return if found.nil?
     content_type :json
-    JSON.pretty_generate nil
+    JSON.pretty_generate found
   end
 
   # Spawn a new thread for this new job
@@ -83,23 +98,25 @@ class RestFtpDaemon < Sinatra::Base
   def process_job
     # Init
     info "process_job: starting"
-    context = Thread.current[:context]
+    job = Thread.current.job
+    job_status :started
     transferred = 0
 
     # Check source
-    job_source = File.expand_path(context["source"])
+    job_source = File.expand_path(job["source"])
     if !(File.exists? job_source)
-      job_status ERR_JOB_SOURCE_NOTFOUND, :ERR_JOB_SOURCE_NOTFOUND
+      job_error ERR_JOB_SOURCE_NOTFOUND, :ERR_JOB_SOURCE_NOTFOUND
       return
     end
     info "process_job: job_source: #{job_source}"
     source_size = File.size job_source
+    job_set :source_size, source_size
 
     # Check target
-    job_target = context["target"]
+    job_target = job["target"]
     target = URI(job_target) rescue nil
     if job_target.nil? || target.nil?
-      job_status ERR_JOB_TARGET_UNPARSEABLE, :ERR_JOB_TARGET_UNPARSEABLE
+      job_error ERR_JOB_TARGET_UNPARSEABLE, :ERR_JOB_TARGET_UNPARSEABLE
       return
     end
     info "process_job: job_target: #{job_target}"
@@ -117,77 +134,102 @@ class RestFtpDaemon < Sinatra::Base
     ftp.login
     ftp.chdir(target_path)
 
+
+    # Check if target file is found
+    info "source: checking target file"
+    job_status :checking_target
+    job_error ERR_BUSY, :checking_target
+
+    results = ftp.list(target_name)
+    info "ftp.list: #{results}"
+    unless results.count.zero?
+      job_error ERR_JOB_TARGET_PRESENT, :ERR_JOB_TARGET_PRESENT
+      info "target: existing: ERR_JOB_TARGET_PRESENT"
+      ftp.close
+      return
+    end
+
+
      # Do transfer
     info "source: starting stransfer"
-    Thread.current[:status] = :transferring
-    job_status ERR_BUSY, :running
-    job_set :source_size, source_size
+    #Thread.current[:status] = :transferring
+    job_status :uploading
+    job_error ERR_BUSY, :uploading
 
     begin
       ftp.putbinaryfile(job_source, target_name, TRANSFER_CHUNK_SIZE) do |block|
         # Update thread info
         percent = (100.0 * transferred / source_size).round(1)
-        info "transferring [#{percent} %] of [#{target_name}]"
         job_set :progress, percent
         job_set :transferred, transferred
+        info "transferring [#{percent} %] of [#{target_name}]"
 
         # Update counters
         transferred += TRANSFER_CHUNK_SIZE
       end
 
     rescue Net::FTPPermError
-      Thread.current[:status] = :failed
-      job_status ERR_JOB_PERMISSION, :ERR_JOB_PERMISSION
+      #job_status :failed
+      job_error ERR_JOB_PERMISSION, :ERR_JOB_PERMISSION
       info "source: FAILED: PERMISSIONS ERROR"
 
     else
-      Thread.current[:status] = :finished
-      job_status ERR_OK, :finished
+      #job_status :finished
+      job_error ERR_OK, :finished
       info "source: finished stransfer"
     end
 
+    # Close FTP connexion
     ftp.close
   end
 
   def get_status
     info "> get_status"
-
     {
     app_name: APP_NAME,
     hostname: @@hostname,
     version: APP_VER,
     started: APP_STARTED,
     uptime: (Time.now - APP_STARTED).round(1),
-    jobs_count: @@threads.list.count,
+    jobs_count: @@workers.list.count,
     }
   end
 
   def get_jobs
     info "> get_jobs"
 
-    output = []
-    @@threads.list.each do |thread|
-      output << {
-      :id => thread[:name],
-      :process => thread.status,
-      :status =>  thread[:status],
-      :context => thread[:context],
-      }
-    end
-    output
+    # Collect info's
+    @@workers.list.map { |thread| thread.job }
   end
 
+  def delete_job id
+    info "> delete_job(#{id})"
 
-  def delete_job name
-    info "> delete_job(#{name})"
+    # Find jobs with this id
+    jobs = jobs_with_id id
 
-    count = 0
-    @@threads.list.collect do |thread|
-      next unless thread[:name] == name
-      Thread.kill(thread)
-      count += 1
-    end
-    count
+    # Kill them
+    jobs.each{ |thread| Thread.kill(thread) }
+
+    # Return the first one
+    return nil if jobs.empty?
+    jobs.first.job
+  end
+
+  def find_job id
+    info "> find_job(#{id})"
+
+    # Find jobs with this id
+    jobs = jobs_with_id id
+
+    # Return the first one
+    return nil if jobs.empty?
+    jobs.first.job
+  end
+
+  def jobs_with_id id
+    info "> find_jobs_by_id(#{id})"
+    @@workers.list.select{ |thread| thread[:id].to_s == id.to_s }
   end
 
   def new_job context = {}
@@ -196,15 +238,15 @@ class RestFtpDaemon < Sinatra::Base
     # Generate name
     @@last_worker_id +=1
     host = @@hostname.split('.')[0]
-    name = "#{host}-#{Process.pid.to_s}-#{@@last_worker_id}"
-    info "new_job: creating thread [#{name}]"
+    worker_id = @@last_worker_id
+    worker_name = "#{host}-#{Process.pid.to_s}-#{worker_id}"
+    info "new_job: creating thread [#{worker_name}]"
 
     # Parse parameters
     job_source = context["source"]
     job_target = context["target"]
     return { code: ERR_REQ_SOURCE_MISSING, errmsg: :ERR_REQ_SOURCE_MISSING} if job_source.nil?
     return { code: ERR_REQ_TARGET_MISSING, errmsg: :ERR_REQ_TARGET_MISSING} if job_target.nil?
-    #return { code: ERR_TRX_SOURCE_FILE_NOT_FOUND, errmsg: "ERR_TRX_SOURCE_FILE_NOT_FOUND [#{job_source}]"} unless File.exists? job_source
 
     # Parse dest URI
     target = URI(job_target)
@@ -212,47 +254,51 @@ class RestFtpDaemon < Sinatra::Base
     return { code: ERR_REQ_TARGET_SCHEME, errmsg: :ERR_REQ_TARGET_SCHEME} unless target.scheme == "ftp"
 
     # Create thread
-    job = Thread.new(name, job) do
-      # Initialize context
-      Thread.current[:name] = name
-      Thread.current[:created] = Time.now.to_f;
-      Thread.current[:status] = :thread_initializing
-
-      # Store job info
-      Thread.current[:context] = context
-      job_status ERR_BUSY, :thread_initializing
+    job = Thread.new(worker_id, worker_name, job) do
+      # Tnitialize thread
       Thread.abort_on_exception = true
+      job_status :initializing
+      job_error ERR_OK
+
+      # Initialize job info
+      Thread.current[:job] = {}
+      Thread.current[:job].merge! context if context.is_a? Enumerable
+      Thread.current[:id] = worker_id
+      job_set :worker_name, worker_name
+      job_set :created, Time.now
 
       # Do the job
       info "new_job: thread running"
       process_job
 
       # Sleep a few seconds before dying
-      Thread.current[:status] = :thread_ending
+      job_status :graceful_ending
       sleep THREAD_SLEEP_BEFORE_DIE
+      job_status :ended
       info "new_job: thread finished"
     end
 
     # Keep thread in thread group
-    info "new_job: attaching thread [#{name}] to group"
-    @@threads.add job
+    info "new_job: attaching thread [#{worker_name}] to group"
+    @@workers.add job
 
-    return { code: 0, errmsg: 'success', name: name, context: context }
+    return { code: 0, errmsg: 'success', worker_id: worker_id, context: context }
   end
 
   def info msg=""
-    @logger.send :info, msg
+    @logger.info msg
   end
 
-  def job_status code, errmsg
-    Thread.current[:context] ||= {}
-    Thread.current[:context][:code] = code
-    Thread.current[:context][:errmsg] = errmsg
+  def job_error error, errmsg = nil
+    job_set :error, error
+    job_set :errmsg, errmsg
+  end
+  def job_status status
+    job_set :status, status
   end
 
-  def job_set attribute, value
-    Thread.current[:context] ||= {}
-    Thread.current[:context][attribute] = value
+  def job_set attribute, value, thread = Thread.current
+    thread[:job][attribute] = value if thread[:job].is_a? Enumerable
   end
 
 

data/rest-ftp-daemon.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = "rest-ftp-daemon"
-  s.version = "0.5.0"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Bruno"]
-  s.date = "2014-07-31"
-  s.description = "A fairly basic FTP client daemon, driven by RESTful webservice calls"
+  s.date = "2014-08-01"
+  s.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
   s.email = "rest-ftp-daemon@bmconseil.com"
   s.executables = ["rest-ftp-daemon"]
   s.extra_rdoc_files = [
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
     "lib/config.rb",
     "lib/config.ru",
     "lib/errors.rb",
+    "lib/extend_threads.rb",
     "lib/rest-ftp-daemon.rb",
     "rest-ftp-daemon.gemspec",
     "test/helper.rb",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.9.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-31 00:00:00.000000000 Z
+date: 2014-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -91,7 +91,8 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.0.1
-description: A fairly basic FTP client daemon, driven by RESTful webservice calls
+description: This is a pretty simple FTP client daemon, controlled through a RESTfull
+  API
 email: rest-ftp-daemon@bmconseil.com
 executables:
 - rest-ftp-daemon
@@ -110,6 +111,7 @@ files:
 - lib/config.rb
 - lib/config.ru
 - lib/errors.rb
+- lib/extend_threads.rb
 - lib/rest-ftp-daemon.rb
 - rest-ftp-daemon.gemspec
 - test/helper.rb
@@ -129,7 +131,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4110508232365468259
+      hash: 3650939871823663291
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: