rest-ftp-daemon 0.6 → 0.9.0

data/Gemfile

@@ -1,3 +1,10 @@
 source "http://rubygems.org"
 
-gemspec
+gem "sinatra"
+gem "json"
+
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 2.0.1"
+end

data/Gemfile.lock

@@ -1,78 +1,81 @@
-PATH
-  remote: .
-  specs:
-    rest-ftp-daemon (0.55)
-      facter
-      grape
-      json
-      settingslogic
-      thin (~> 1.6)
-
 GEM
   remote: http://rubygems.org/
   specs:
-    CFPropertyList (2.2.8)
-    activesupport (4.1.6)
+    activesupport (4.1.4)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.1)
       tzinfo (~> 1.1)
-    axiom-types (0.1.1)
-      descendants_tracker (~> 0.0.4)
-      ice_nine (~> 0.11.0)
-      thread_safe (~> 0.3, >= 0.3.1)
+    addressable (2.3.6)
     builder (3.2.2)
-    coercible (1.0.0)
-      descendants_tracker (~> 0.0.1)
-    daemons (1.1.9)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
-    equalizer (0.0.9)
-    eventmachine (1.0.3)
-    facter (2.2.0)
-      CFPropertyList (~> 2.2.6)
-    grape (0.9.0)
-      activesupport
-      builder
-      hashie (>= 2.1.0)
-      multi_json (>= 1.3.2)
-      multi_xml (>= 0.5.2)
-      rack (>= 1.3.0)
-      rack-accept
-      rack-mount
-      virtus (>= 1.0.0)
-    hashie (3.3.1)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.8)
+    github_api (0.12.0)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.3)
+      oauth2
+    hashie (3.2.0)
+    highline (1.6.21)
     i18n (0.6.11)
-    ice_nine (0.11.0)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
     json (1.8.1)
-    minitest (5.4.1)
+    jwt (1.0.0)
+    mini_portile (0.6.0)
+    minitest (5.4.0)
     multi_json (1.10.1)
     multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
     rack (1.5.2)
-    rack-accept (0.4.5)
-      rack (>= 0.4)
-    rack-mount (0.8.3)
-      rack (>= 1.0.0)
+    rack-protection (1.5.3)
+      rack
     rake (10.3.2)
-    settingslogic (2.0.9)
-    thin (1.6.2)
-      daemons (>= 1.0.9)
-      eventmachine (>= 1.0.0)
-      rack (>= 1.0.0)
+    rdoc (4.1.1)
+      json (~> 1.4)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.6.2)
+      activesupport (>= 3.0.0)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
     thread_safe (0.3.4)
-    tzinfo (1.2.2)
+    tilt (1.4.1)
+    tzinfo (1.2.1)
       thread_safe (~> 0.1)
-    virtus (1.0.3)
-      axiom-types (~> 0.1)
-      coercible (~> 1.0)
-      descendants_tracker (~> 0.0, >= 0.0.3)
-      equalizer (~> 0.0, >= 0.0.9)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.6)
-  rake
-  rest-ftp-daemon!
+  bundler (~> 1.0)
+  jeweler (~> 2.0.1)
+  json
+  shoulda
+  sinatra

data/README.md

@@ -1,46 +1,25 @@
 rest-ftp-daemon
 ====================================================================================
 
-
-
 This is a pretty simple FTP client daemon, controlled through a RESTfull API.
 
 As of today, its main features are :
 
-* Allow environment-specific configuration in a YAML file
-* Delegate a transfer job by ``POST```'ing a simple JSON structure
+* Delegate a transfer job, ```PUT```'ing posting simple JSON structure
 * Spawn a dedicated thread to handle this job in its own context
-* Report transfer status, progress and errors for each job in realtime
+* Report transfer status, progress and errors for each delegated job
 * Expose JSON status of workers on ```GET /jobs/``` for automated monitoring
-* Parralelize jobs as soon as they arrive
-* Handle job queues and priority as an attribute of the job
-* Allow dynamic evaluation of priorities, and change of any attribute until the job is picked
-* Provide RESTful notifications to the requesting client
-* Allow authentication in FTP target in a standard URI-format
-* Allow configuration-based path templates to abstract local mounts or remote FTPs (endpoint tokens)
-
-
-Expected features in a short-time range :
-
-* Allow change of priorities or other attributes after a job has been started
-* Offer a basic dashboard directly within the daemon HTTP interface
-* Periodically send an update-notification with transfer status and progress
-* Allow fallback file source when first file path is unavailable (failover)
-* Provide swagger-style API documentation
-* Authenticate API clients
-
 
 
 Installation
 ------------------------------------------------------------------------------------
 
-This project is available as a rubygem, requires on ruby >= 1.9.3 and rubygems installed.
+This project is available as a rubygem, requires on ruby >= 1.9 and rubygems installed.
 
 Get and install the gem from rubygems.org:
 
 ```
-# apt-get install ruby1.9.3 ruby-dev rubygems gcc g++
-gem install rest-ftp-daemon --no-ri --no-rdoc
+gem install rest-ftp-daemon
 ```
 
 Start the daemon:
@@ -49,117 +28,49 @@ Start the daemon:
 rest-ftp-daemon start
 ```
 
-Start the daemon on a specific port :
-
-```
-rest-ftp-daemon -p4000 start
-```
-
 Check that the daemon is running and giving status info
 
 ```
 http://localhost:3000/
 ```
 
-Configuration
-------------------------------------------------------------------------------------
-Most of the configuration options live in a YAML configuration file, containing two main sections:
-
-* the ``defaults`` section should be left as-is and will be used is no other environment-specific value is provided.
-* the ``production`` section can receive personnalized settings according to your environment-specific setup and paths.
-
-Configuration priority is defined as follows (from most important to last resort):
-
-* command-line parameters
-* config file defaults section
-* config file environment section
-* application internal defaults
-
-
-As a starting point, ``rest-ftp-daemon.yml.sample`` is an exemple config file that can be  copied into the expected location ``/etc/rest-ftp-daemon.yml``.
-
-
-Logging
-------------------------------------------------------------------------------------
-
-The application will not log to any file by default, if not specified in its configuration.
-Otherwise separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages. Providing and empty value will simply activate logging to ``STDOUT``.
+For now, daemon logs to ```APP_LOGTO``` defined in ```lib/config.rb```
 
 
 Usage examples
 ------------------------------------------------------------------------------------
 
-* Start a job to transfer a file named "file.iso" to a local FTP server
+Start a job to transfer a file named "file.ova" to a local FTP server
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
+'{"source":"~/file.ova","target":"ftp://anonymous@localhost/incoming/dest2.ova"}' "http://localhost:3000/jobs"
 ```
 
-Requesting notifications is achieved by passing a "notify" key in the request, with a callback URL.
-This URL will be called at some points, ``POST```'ing a generic JSON structure with progress information.
-
-
-* Start a job requesting notifications ``POST```'ed on "http://requestb.in/1321axg1"
+Start a job to transfer a file named "file.dmg" to a local FTP server
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
+'{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg"}' "http://localhost:3000/jobs"
 ```
 
-* Start a job with all the above plus a priority
+Get status of a specific job based on its name
 
 ```
-curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
+curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
 ```
 
-* Start a job using endpoint tokens
-
-First define ``nas`` ans ``ftp1`` in the configuration file :
+Delete a specific job based on its name
 
 ```
-defaults: &defaults
-
-development:
-  <<: *defaults
-
-  endpoints:
-    nas: "~/"
-    ftp1: "ftp://anonymous@localhost/incoming/"
-```
-
-Thos tokens will be expanded when the job is ran :
-
-```
-curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
-'{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
-```
-
-NB: a special token [RANDOM] helps to generate a random filename when needed
-
-* Get status of a specific job based on its ID
-
-```
-curl -H "Content-Type: application/json" -X GET -D /dev/stdout "http://localhost:3000/jobs/3"
-```
-
-
-* Delete a specific job based on its ID
-
-```
-curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/3"
+curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
 ```
 
 
 Getting status
 ------------------------------------------------------------------------------------
 
-* A global JSON status is provided on ``` GET /index.json ```
-
-* A nice dashboard gives a global view of the daemon, jobs in queue, and system status, exposed on ``` GET /```
-
-* The server exposes its jobs list on ``` GET /jobs ```
+The server exposes jobs list on ``` GET /jobs ```
 
 ```
 http://localhost:3000/jobs
@@ -184,19 +95,18 @@ This query will return a job list :
     "transferred": 37100000
   },
   {
-    source: "[nas]/file.dmg",
-    target: "[ftp2]/dest4.dmg",
-    notify: "http://requestb.in/1321axg1",
-    updated_at: "2014-09-17 22:56:14 +0200",
-    id: 2,
-    started_at: "2014-09-17 22:56:01 +0200",
-    status: "uploading",
-    error: 0,
-    debug_source: "/Users/bruno/file.dmg",
-    debug_target: "#<URI::FTP:0x007ffa9289e650 URL:ftp://uuuuuuuu:yyyyyyyyy@ftp.xxxxxx.fr/subdir/dest4.dmg>",
-    file_size: 32093208,
-    file_progress: 5.6,
-    file_sent: 1800000
+    "source": "~/file.ova",
+    "target": "ftp://anonymous@localhost/incoming/dest2.ova",
+    "worker_name": "bob-92439-2",
+    "created": "2014-08-01 16:53:12 +0200",
+    "id": 17,
+    "runtime": 13.8,
+    "status": "uploading",
+    "source_size": 1849036800,
+    "error": -1,
+    "errmsg": "uploading",
+    "progress": 36.1,
+    "transferred": 668300000
   }
 ]
 ```

data/Rakefile

@@ -1,3 +1,51 @@
 # encoding: utf-8
-require "bundler/gem_tasks"
+
 require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "rest-ftp-daemon"
+  gem.homepage = "http://github.com/bmedici/rest-ftp-daemon"
+  gem.license = "MIT"
+  gem.summary = "RESTful FTP client daemon"
+  gem.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
+  gem.email = "rest-ftp-daemon@bmconseil.com"
+  gem.authors = ["Bruno"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "rest-ftp-daemon #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.9.0

data/bin/rest-ftp-daemon

@@ -1,42 +1,22 @@
 #!/usr/bin/env ruby
 
 # Libs and init
-require "thin"
-DEVELOPMENT=ARGV.include?("development")
-HERE=File.expand_path(File.dirname(__FILE__) + '/../')
-#APP_NAME2="rest-ftp-daemon"
-
-# Include config file and build rack commandline
-require File.expand_path("#{HERE}/lib/rest-ftp-daemon/config.rb")
-
+require 'thin'
+
+# Initialize some local constants
+APP_ROOT = File.dirname(__FILE__) + '/../'
+APP_NAME = 'rest-ftp-daemon'
+APP_VER = File.read "#{APP_ROOT}/VERSION"
+APP_STARTED = Time.now
+APP_DEFAULT_PORT = 3000
+APP_LOGTO = "/tmp/#{APP_NAME}.log"
+
+# Prepare thin
+rackup_file = File.expand_path "#{APP_ROOT}/lib/config.ru"
 argv = ARGV
-argv << ["-p", Settings.port.to_s] unless ARGV.include?("-p") || Settings.port.nil?
-argv << ["-e", Settings.namespace] unless ARGV.include?("-e") || Settings.namespace.nil?
-argv << ["-l", Settings.logs.thin] unless ARGV.include?("-l") unless Settings.logs.thin.nil? unless Settings.logs.nil?
-argv << ["--daemonize"] if (Settings.daemonize==1 || Settings.daemonize.nil?) unless ARGV.include?("-d")
-
-# Rackup file
-argv << ["-R", File.expand_path("#{HERE}/config.ru")] unless ARGV.include?("-R")
-
-# Display compiled configuration
-puts
-puts "Daemon name \t #{Settings.name}"
-puts "Version     \t #{Settings.version}"
-puts "Environment \t #{Settings.namespace}"
-puts "Config file \t #{Settings.source}"
-puts "Parameters  \t #{argv.flatten}"
-puts Settings.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion => false )
-
-# Start Thin with this rackup configuration
-puts
-begin
-  Thin::Runner.new(argv.flatten).run!
-rescue Thin::PidFileExist
-  puts "EXITING: daemon was already running (Thin::PidFileExist)"
-rescue Thin::PidFileNotFound
-  puts "EXITING: daemon was not running (Thin::PidFileNotFound)"
-else
-  puts "PROCESS ENDING"
-end
-
+argv << ["-R", rackup_file] unless ARGV.include?("-R")
+argv << ["-p", APP_DEFAULT_PORT.to_s] unless ARGV.include?("-p")
+argv << ["-e", "production"] unless ARGV.include?("-e")
+argv << ["--stats", "/stats"] unless ARGV.include?("--stats")
 
+Thin::Runner.new(argv.flatten).run!

data/lib/config.rb

@@ -0,0 +1,3 @@
+# Transfer configuration
+TRANSFER_CHUNK_SIZE = 100000
+THREAD_SLEEP_BEFORE_DIE = 120

data/lib/config.ru

@@ -0,0 +1,11 @@
+# Main libs
+require 'sinatra'
+require 'sinatra/base'
+require 'net/ftp'
+require 'json'
+
+# My local libs
+Dir[APP_ROOT+"/lib/*.rb"].each {|file| require File.expand_path file }
+
+# Start application
+run RestFtpDaemon

data/lib/errors.rb

@@ -0,0 +1,12 @@
+ERR_OK                            = 0
+ERR_BUSY                          = -1
+
+ERR_REQ_SOURCE_MISSING            = 11
+ERR_REQ_TARGET_MISSING            = 12
+ERR_REQ_TARGET_SCHEME             = 13
+
+ERR_JOB_SOURCE_NOTFOUND            = 21
+ERR_JOB_TARGET_UNPARSEABLE         = 22
+ERR_JOB_PERMISSION                 = 24
+ERR_JOB_TARGET_PRESENT             = 25
+

data/lib/extend_threads.rb

@@ -0,0 +1,14 @@
+class Thread
+
+  def job
+    # Basic fields
+    job = self[:job]
+
+    # Calculated fields
+    job[:id] = self[:id]
+    job[:runtime] = (Time.now - job[:created]).round(1)
+
+    return job
+  end
+
+end