rest-ftp-daemon 0.41 → 0.55

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDQwNWFlNDE3ZGUwZGY2ZTc1N2FiMGE3OWNkODMxN2VhODdkNDNiMA==
+    YjFjZjg2NTk1Y2M5NzVmNGM4MWJmYjA1YTEwN2JhZjVjOTlkN2I2OQ==
   data.tar.gz: !binary |-
-    MmM2YmRkMDkwMjM2OTM2YjZlOTcyNjU0NmNmODQwNmJhNjI0YzdlNQ==
+    OGViZWY5NzFjNjI2NDRlYzVkOTNhNjliY2UxMDg3MTRmMTFhMTVmMA==
 SHA512:
   metadata.gz: !binary |-
-    YzEzZjdmYzEyNWJhZTRkODc0NmY4NjRjNTMxNmZiZWFmMWIwOTg5YmM3YzE5
-    NjIwYTNlMTgzNzhhYTljYTUzNTk2NTk3YzJmOTgzYmNjNzAwY2NiMjFlMDYz
-    MjU3ZmZmNjBhYTA0ZWUzOWJkY2MyNWI1ZmYyM2M5ZDVmYWNiNTA=
+    ODExM2M0MjhhOTE2OGRjNTIyNmFlZjU5MzkxZTY2YzVkMjRmMDljYzhmZDRl
+    M2ZmZWVjZDE5NjYzNTM3NzJiMGVkOGQ3YTJkZjJkYzE3Yzc0ODk5M2M3MmRm
+    MWIzNTk3ZDc5ODc5YWJjZTY3MGRhNzhlMTNiYTA5NTRiYzhhNDQ=
   data.tar.gz: !binary |-
-    ZTEyZmVkYjIwYzQ2NjVkZjEyNTUzOGFjY2NkNGJjMzkyZWIzZDQ4ZWJiOGNi
-    OGUxMGY5Nzk4YzUxZWEzMjE5Mjg3NDhkMDI1MzFjMWYyNjYxMDk2NmYzZDY2
-    ODQ0NzIwMWY1NGQwYWZjYTE1NjRiMjFhNGZiYzI1Y2U2YjEwMjI=
+    NWIxNTNlNzk5NDY3NGQ1ZjI4NzY1ZDlmMWI0NTRkYzU2NDRiMzg1YmZlODVi
+    NTlhNzVkNDk4Yjc5MzdiMjgwMTExNzA4MGQwNGZhZDdmZTA5ZGY4Njg2MTUz
+    OWMxNDA4NjQwMzUxZTYxNGFhM2FkYjRjY2U4NTFiZmQ0NGEzZGU=

data/.gitignore

@@ -1,8 +1,6 @@
 pkg
-NOTES.txt
 .bundle
 .DS_Store
 *.log
 tmp/
-
-
+work/

data/Gemfile.lock

@@ -1,15 +1,18 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.30.1)
+    rest-ftp-daemon (0.55)
+      facter
       grape
       json
+      settingslogic
       thin (~> 1.6)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    activesupport (4.1.5)
+    CFPropertyList (2.2.8)
+    activesupport (4.1.6)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -27,6 +30,8 @@ GEM
       thread_safe (~> 0.3, >= 0.3.1)
     equalizer (0.0.9)
     eventmachine (1.0.3)
+    facter (2.2.0)
+      CFPropertyList (~> 2.2.6)
     grape (0.9.0)
       activesupport
       builder
@@ -50,6 +55,7 @@ GEM
     rack-mount (0.8.3)
       rack (>= 1.0.0)
     rake (10.3.2)
+    settingslogic (2.0.9)
     thin (1.6.2)
       daemons (>= 1.0.9)
       eventmachine (>= 1.0.0)

data/README.md

@@ -1,38 +1,36 @@
 rest-ftp-daemon
 ====================================================================================
 
+
+
 This is a pretty simple FTP client daemon, controlled through a RESTfull API.
 
 As of today, its main features are :
 
+* Allow environment-specific configuration in a YAML file
 * Delegate a transfer job by ``POST```'ing a simple JSON structure
 * Spawn a dedicated thread to handle this job in its own context
 * Report transfer status, progress and errors for each job in realtime
 * Expose JSON status of workers on ```GET /jobs/``` for automated monitoring
 * Parralelize jobs as soon as they arrive
+* Handle job queues and priority as an attribute of the job
+* Allow dynamic evaluation of priorities, and change of any attribute until the job is picked
+* Provide RESTful notifications to the requesting client
+* Allow authentication in FTP target in a standard URI-format
+* Allow configuration-based path templates to abstract local mounts or remote FTPs (endpoint tokens)
+
 
 Expected features in a short-time range :
 
-* Handle job queues
-* Handle job priorities
 * Allow change of priorities or other attributes after a job has been started
-* Provide RESTful notifications to the requesting client
 * Offer a basic dashboard directly within the daemon HTTP interface
 * Periodically send an update-notification with transfer status and progress
 * Allow fallback file source when first file path is unavailable (failover)
-* Some refactoring may be needed after thos steps
 * Provide swagger-style API documentation
 * Authenticate API clients
 
 
 
-Documentation TODO
-------------------------------------------------------------------------------------
-overwrite: any non empty value allows overwriting
-todo: queues
-
-
-
 Installation
 ------------------------------------------------------------------------------------
 
@@ -51,51 +49,117 @@ Start the daemon:
 rest-ftp-daemon start
 ```
 
+Start the daemon on a specific port :
+
+```
+rest-ftp-daemon -p4000 start
+```
+
 Check that the daemon is running and giving status info
 
 ```
 http://localhost:3000/
 ```
 
-For now, daemon logs to ```APP_LOGTO``` defined in ```lib/config.rb```
+Configuration
+------------------------------------------------------------------------------------
+Most of the configuration options live in a YAML configuration file, containing two main sections:
 
+* the ``defaults`` section should be left as-is and will be used is no other environment-specific value is provided.
+* the ``production`` section can receive personnalized settings according to your environment-specific setup and paths.
 
-Usage examples
+Configuration priority is defined as follows (from most important to last resort):
+
+* command-line parameters
+* config file defaults section
+* config file environment section
+* application internal defaults
+
+
+As a starting point, ``rest-ftp-daemon.yml.sample`` is an exemple config file that can be  copied into the expected location ``/etc/rest-ftp-daemon.yml``.
+
+
+Logging
 ------------------------------------------------------------------------------------
 
-Requesting notifications is achieved by passing a "notify" key in the request, with a callback URL. This URL will be called at some points, ``POST```'ing a generic JSON structure with progress information.
+The application will not log to any file by default, if not specified in its configuration.
+Otherwise separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages. Providing and empty value will simply activate logging to ``STDOUT``.
+
+
+Usage examples
+------------------------------------------------------------------------------------
 
-Start a job to transfer a file named "file.iso" to a local FTP server
+* Start a job to transfer a file named "file.iso" to a local FTP server
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 '{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
 ```
 
-Start a job to transfer a file, and request notifications ``POST```'ed on "http://requestb.in/1321axg1"
+Requesting notifications is achieved by passing a "notify" key in the request, with a callback URL.
+This URL will be called at some points, ``POST```'ing a generic JSON structure with progress information.
+
+
+* Start a job requesting notifications ``POST```'ed on "http://requestb.in/1321axg1"
 
 ```
 curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 '{"source":"~/file.dmg","target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
 ```
 
-Get status of a specific job based on its name
+* Start a job with all the above plus a priority
+
+```
+curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+'{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
+```
+
+* Start a job using endpoint tokens
+
+First define ``nas`` ans ``ftp1`` in the configuration file :
 
 ```
-curl -H "Content-Type: application/json" -X GET -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+defaults: &defaults
+
+development:
+  <<: *defaults
+
+  endpoints:
+    nas: "~/"
+    ftp1: "ftp://anonymous@localhost/incoming/"
 ```
 
-Delete a specific job based on its name
+Thos tokens will be expanded when the job is ran :
 
 ```
-curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/bob-45320-1"
+curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+'{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
+```
+
+NB: a special token [RANDOM] helps to generate a random filename when needed
+
+* Get status of a specific job based on its ID
+
+```
+curl -H "Content-Type: application/json" -X GET -D /dev/stdout "http://localhost:3000/jobs/3"
+```
+
+
+* Delete a specific job based on its ID
+
+```
+curl -H "Content-Type: application/json" -X DELETE -D /dev/stdout "http://localhost:3000/jobs/3"
 ```
 
 
 Getting status
 ------------------------------------------------------------------------------------
 
-The server exposes jobs list on ``` GET /jobs ```
+* A global JSON status is provided on ``` GET /index.json ```
+
+* A nice dashboard gives a global view of the daemon, jobs in queue, and system status, exposed on ``` GET /```
+
+* The server exposes its jobs list on ``` GET /jobs ```
 
 ```
 http://localhost:3000/jobs
@@ -120,18 +184,19 @@ This query will return a job list :
     "transferred": 37100000
   },
   {
-    "source": "~/file.ova",
-    "target": "ftp://anonymous@localhost/incoming/dest2.ova",
-    "worker_name": "bob-92439-2",
-    "created": "2014-08-01 16:53:12 +0200",
-    "id": 17,
-    "runtime": 13.8,
-    "status": "uploading",
-    "source_size": 1849036800,
-    "error": -1,
-    "errmsg": "uploading",
-    "progress": 36.1,
-    "transferred": 668300000
+    source: "[nas]/file.dmg",
+    target: "[ftp2]/dest4.dmg",
+    notify: "http://requestb.in/1321axg1",
+    updated_at: "2014-09-17 22:56:14 +0200",
+    id: 2,
+    started_at: "2014-09-17 22:56:01 +0200",
+    status: "uploading",
+    error: 0,
+    debug_source: "/Users/bruno/file.dmg",
+    debug_target: "#<URI::FTP:0x007ffa9289e650 URL:ftp://uuuuuuuu:yyyyyyyyy@ftp.xxxxxx.fr/subdir/dest4.dmg>",
+    file_size: 32093208,
+    file_progress: 5.6,
+    file_sent: 1800000
   }
 ]
 ```

data/bin/rest-ftp-daemon

@@ -2,29 +2,37 @@
 
 # Libs and init
 require "thin"
-
-# Initialize some local constants
 APP_ROOT = File.dirname(__FILE__) + '/../'
-rackup_file = File.expand_path "#{APP_ROOT}/config.ru"
-
-# Include config file
-require File.expand_path "#{APP_ROOT}/lib/rest-ftp-daemon/config.rb"
+DEVELOPMENT = ARGV.include?("development")
 
-# Prepare thin
+# Include config file and build rack commandline
+require File.expand_path("#{APP_ROOT}/lib/rest-ftp-daemon/config.rb")
 argv = ARGV
-argv << ["-R", rackup_file] unless ARGV.include?("-R")
-argv << ["-l", "/tmp/thin.log"] unless ARGV.include?("-l")
-argv << ["--daemonize"] unless ARGV.include?("--daemonize")
-argv << ["-e", "production"] unless ARGV.include?("-e")
-argv << ["-p", (RestFtpDaemon::PORT).to_s] unless ARGV.include?("-p")
-#argv << ["--stats", "/stats"] unless ARGV.include?("--stats")
+argv << ["-p", Settings.port.to_s] unless ARGV.include?("-p") || Settings.port.nil?
+argv << ["-e", Settings.namespace] unless ARGV.include?("-e") || Settings.namespace.nil?
+argv << ["-l", Settings.logs.thin] unless ARGV.include?("-l") unless Settings.logs.thin.nil? unless Settings.logs.nil?
+argv << ["--daemonize"] if (Settings.daemonize==1 || Settings.daemonize.nil?) unless ARGV.include?("-d")
+
+# Rackup file
+argv << ["-R", File.expand_path("#{APP_ROOT}/config.ru")] unless ARGV.include?("-R")
+
+# Display compiled configuration
+puts
+puts "Daemon name \t #{Settings.name}"
+puts "Version     \t #{Settings.version}"
+puts "Environment \t #{Settings.namespace}"
+puts "Config file \t #{Settings.source}"
+puts "Parameters  \t #{argv.flatten}"
+puts Settings.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion => false )
 
 # Start Thin with this rackup configuration
-puts "#{RestFtpDaemon::NAME}: using [#{argv.join (' ')}]"
+puts
 begin
   Thin::Runner.new(argv.flatten).run!
+rescue Thin::PidFileExist
+  puts "EXITING: daemon was already running (Thin::PidFileExist)"
 rescue Thin::PidFileNotFound
-  puts "#{RestFtpDaemon::NAME}: not running (Thin::PidFileNotFound)"
+  puts "EXITING: daemon was not running (Thin::PidFileNotFound)"
 end
 
 

data/config.ru

@@ -7,8 +7,9 @@ APP_STARTED = Time.now
 
 # Create worker pool
 $queue = RestFtpDaemon::JobQueue.new
-$pool = RestFtpDaemon::WorkerPool.new(1)
+$pool = RestFtpDaemon::WorkerPool.new(Settings.workers.to_i)
 
 # Start REST FTP Daemon
 #run Rack::Cascade.new [Rack::File.new("/public"), RestFtpDaemon::API::Root]
+#run Rack::URLMap.new("/" => Frontend.new, "/api" => Api.new)
 run Rack::Cascade.new [RestFtpDaemon::API::Root]

data/lib/rest-ftp-daemon.rb

@@ -1,8 +1,6 @@
 # Global libs
+require 'rubygems'
 require 'json'
-require 'grape'
-require 'net/ftp'
-require 'net/http'
 require 'securerandom'
 # require 'celluloid/autostart'
 
@@ -18,5 +16,4 @@ require 'rest-ftp-daemon/notification'
 require 'rest-ftp-daemon/api/defaults'
 require 'rest-ftp-daemon/api/jobs'
 require 'rest-ftp-daemon/api/root'
-require 'rest-ftp-daemon/www'
 

data/lib/rest-ftp-daemon/api/defaults.rb

@@ -1,3 +1,5 @@
+require 'grape'
+
 module RestFtpDaemon
   module API
     module Defaults

data/lib/rest-ftp-daemon/api/index.haml

@@ -0,0 +1,105 @@
+!!! 5
+%html{:lang => "en"}
+  %head
+    %meta{:charset => "utf-8"}/
+    %link{ href:"//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css" , rel: "stylesheet"}
+
+    %title="#{Settings.name} dashboard"
+
+    :css
+
+      #footer {
+        padding: 15px 0;
+        _border-top: 1px solid silver;
+        }
+
+      .progress {
+        margin-bottom: 0;
+        }
+
+  %body
+
+    .container
+      %h1
+        = Settings.name
+        %small
+          = "v#{Settings.version}"
+          = "[#{Settings.namespace}]"
+
+
+      %h2 System status
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-warning Processors
+        .btn.btn-default= @info_procs
+
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-warning Load average
+        .btn.btn-default= @info_load.round(1)
+
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-warning Normalized load
+        .btn.btn-default= "#{@info_norm} %"
+
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-success IP Address
+        .btn.btn-default= @info_ipaddr
+
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-success Memory free
+        .btn.btn-default= @info_memfree
+
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-info Settings.workers
+        .btn.btn-default= Settings.workers
+
+
+      %h2
+        Jobs on this system (#{@jobs_all})
+        %small
+          queued (#{@jobs_queued}) /  popped (#{@jobs_popped})
+
+      %table.table.table-striped.table-hover.table-condensed
+        %tr
+          %th= "ID"
+          %th= "priority"
+          %th= "source"
+          %th= "target"
+          %th= "status"
+          %th= "error"
+          %th= "progress"
+
+        - @jobs.each do |job|
+          - error = job.get :error
+          - status = job.get :status
+          - progress = job.get :progress
+
+          - if error!=0 && !error.nil?
+            - trclass = "danger"
+          - elsif status == :uploading
+            - trclass = "info"
+          - elsif status == :finished
+            - trclass = "success"
+          - else
+            - trclass = "warning"
+
+          %tr{class: trclass}
+
+            %td= job.id
+            %td= job.get :priority
+            %td= job.get :source
+            %td= job.get :target
+            %td= status
+            %td= error
+            %td
+              - unless progress.nil?
+                .progress
+                  .progress-bar{style:"width: #{progress}%;"}
+                    = "#{progress} %"
+
+
+      %h2 Endpoint tokens
+      - endpoints = Settings.endpoints || {}
+      - endpoints.each do |token, value|
+        .btn-group.btn-group-sm
+          .btn.btn-default.btn-danger= token
+          .btn.btn-default= value

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -3,7 +3,7 @@ module RestFtpDaemon
 
     class Jobs < Grape::API
       include RestFtpDaemon::API::Defaults
-      logger ActiveSupport::Logger.new APP_LOGTO, 'daily'
+      logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
 
       params do
         optional :overwrite, type: Integer, default: false

data/lib/rest-ftp-daemon/api/root.rb

@@ -1,9 +1,14 @@
+require 'haml'
+require "facter"
+require "sys/cpu"
+
+
 module RestFtpDaemon
   module API
 
     class Root < Grape::API
       include RestFtpDaemon::API::Defaults
-      logger ActiveSupport::Logger.new APP_LOGTO, 'daily'
+      logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
       #add_swagger_documentation
 
       mount RestFtpDaemon::API::Jobs => '/jobs'
@@ -41,12 +46,54 @@ module RestFtpDaemon
 
       # Server global status
       get '/' do
+        # Prepare data
+        @jobs = $queue.all
+        @jobs_all = $queue.all_size
+        @jobs_queued = $queue.queued_size
+        @jobs_popped = $queue.popped_size
+
+        # Initialize UsageWatch
+        Facter.loadfacts
+        @info_load = Sys::CPU.load_avg.first.to_f
+        @info_procs = (Facter.value :processorcount).to_i
+        @info_ipaddr = Facter.value :ipaddress
+        @info_memfree = Facter.value :memoryfree
+
+        # Compute normalized load
+        if @info_procs.zero?
+          @info_norm = "N/A"
+        else
+          @info_norm = (100 * @info_load / @info_procs).round(1)
+        end
+
+        # Compile haml template
+        @name = "Test"
+        template = File.read("#{File.dirname(__FILE__)}/index.haml")
+        haml_engine = Haml::Engine.new(template)
+        output = haml_engine.render(binding)
+        #[200, {'Content-Type' => 'text/html'}, [output]]
+
+        # Generate output
+        # env['api.format'] = :binary
+        # env['api.format'] = :txt
+
+        # Send response
+        env['api.format'] = :html
+        format "html"
+        status 200
+        content_type "text/html"
+        body output
+
+      end
+
+      # Server global status
+      get '/index.json' do
         info "GET /"
         status 200
         return  {
-          name: RestFtpDaemon::NAME,
           hostname: `hostname`.chomp,
-          version: RestFtpDaemon::VERSION,
+          version: Settings.version,
+          config: Settings.to_hash,
           started: APP_STARTED,
           uptime: (Time.now - APP_STARTED).round(1),
           status: job_list_by_status,

data/lib/rest-ftp-daemon/api/workers.rb

@@ -0,0 +1,49 @@
+# module RestFtpDaemon
+#   module API
+
+#     class Workers < Grape::API
+#       include RestFtpDaemon::API::Defaults
+#       logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
+
+#       helpers do
+#         def info message, level = 0
+#           Jobs.logger.add(Logger::INFO, "#{'  '*level} #{message}", "API::Workers")
+#         end
+
+#         def worker_list
+#           return {
+#             busy: $pool.busy_size,
+#             idle: $pool.idle_size,
+#             to_s: $pool.to_s,
+#             }
+#           #return $workers.list.size
+#           $workers.list.map do |thread|
+#             #next unless thread[:job].is_a? Worker
+#             "worker"
+#             #thread[:worker].inspect
+#           end
+#         end
+
+#       end
+
+#       # List jobs
+#       desc "Get a list of workers"
+#       get do
+#         info "GET /workers"
+#         begin
+#           response = worker_list
+#         rescue RestFtpDaemonException => exception
+#           status 501
+#           api_error exception
+#         rescue Exception => exception
+#           status 501
+#           api_error exception
+#         else
+#           status 200
+#           response
+#         end
+#       end
+
+#     end
+#   end
+# end

data/lib/rest-ftp-daemon/common.rb

@@ -6,7 +6,7 @@ module RestFtpDaemon
 
     def initialize
       # Logger
-      @logger = ActiveSupport::Logger.new APP_LOGTO, 'daily'
+      @logger = ActiveSupport::Logger.new Settings.logs.workers, 'daily' unless Settings.logs.workers.nil?
     end
 
     def id

data/lib/rest-ftp-daemon/config.rb

@@ -1,15 +1,24 @@
-module RestFtpDaemon
-  # Global config
-  NAME = 'rest-ftp-daemon'
-  VERSION = "0.41"
-  PORT = 3000
+require 'settingslogic'
+DEVELOPMENT = false unless defined? DEVELOPMENT
+CONFIG_FILE = "/etc/rest-ftp-daemon.yml"
 
-  # Transfer config
-  TRANSFER_CHUNK_SIZE = 100000
-  THREAD_SLEEP_BEFORE_DIE = 10
-
-  # Logging
-  APP_LOGTO = "/tmp/#{NAME}.log"
-  LOG_TRIM_PROGNAME = 18
+class Settings < Settingslogic
+  namespace DEVELOPMENT ? "development" : "production"
+  suppress_errors namespace!="development"
+end
 
+# Fix application defaults and load config file if found
+if File.exists? CONFIG_FILE
+  Settings.source CONFIG_FILE
+else
+  Settings.source Hash.new
 end
+
+# Forced shared settings
+Settings[:name] = "rest-ftp-daemon"
+Settings[:version] = 0.55
+
+# Forced fixed settings
+Settings[:default_trim_progname] = "18"
+Settings[:default_chunk_size] = "1000000"
+

data/lib/rest-ftp-daemon/job.rb

@@ -1,3 +1,6 @@
+require 'net/ftp'
+
+
 module RestFtpDaemon
   class Job < RestFtpDaemon::Common
 
@@ -10,22 +13,15 @@ module RestFtpDaemon
       @target = nil
       @source = nil
 
-      # Logger
-      #@logger = ActiveSupport::Logger.new APP_LOGTO, 'daily'
-
       # Init context
       set :id, id
       set :started_at, Time.now
-      set :status, :initialized
+      set :status, :created
 
       # Send first notification
       notify "rftpd.queued"
     end
 
-    # def job_id
-    #   get :id
-    # end
-
     def progname
       job_id = get(:id)
       "JOB #{job_id}"
@@ -99,6 +95,11 @@ module RestFtpDaemon
       @status = text
     end
 
+    def get attribute
+      return nil unless @params.is_a? Enumerable
+      @params[attribute.to_s]
+    end
+
   protected
 
     def up_time
@@ -130,9 +131,28 @@ module RestFtpDaemon
       @params[attribute.to_s] = value
     end
 
-    def get attribute
-      return nil unless @params.is_a? Enumerable
-      @params[attribute.to_s]
+    def expand_path_from path
+      File.expand_path replace_token_in_path(path)
+    end
+
+    def expand_url_from path
+      URI replace_token_in_path(path) rescue nil
+    end
+
+    def replace_token_in_path path
+      # Ensure endpoints are not a nil value
+      return path unless Settings.endpoints.is_a? Enumerable
+      newpath = path.clone
+
+      # Replace endpoints defined in config
+      Settings.endpoints.each do |from, to|
+        newpath.gsub! "[#{from}]", to
+      end
+
+      # Replace with the special RAND token
+      newpath.gsub! "[RANDOM]", SecureRandom.hex(8)
+
+      return newpath
     end
 
     def prepare
@@ -141,15 +161,18 @@ module RestFtpDaemon
 
       # Check source
       raise JobSourceMissing unless @params["source"]
-      @source = File.expand_path(@params["source"])
+      @source = expand_path_from @params["source"]
       set :debug_source, @source
-      raise JobSourceNotFound unless File.exists? @source
 
       # Check target
       raise JobTargetMissing unless @params["target"]
-      @target = URI(@params["target"]) rescue nil
+      @target = expand_url_from @params["target"]
       set :debug_target, @target.inspect
+
+      # Check compliance
       raise JobTargetUnparseable if @target.nil?
+      raise JobSourceNotFound unless File.exists? @source
+
     end
 
     def transfer_fake
@@ -200,20 +223,24 @@ module RestFtpDaemon
 
       # Do transfer
       set :status, :uploading
-      ftp.putbinaryfile(@source, target_name, TRANSFER_CHUNK_SIZE) do |block|
+      chunk_size = Settings.transfer.chunk_size || Settings[:default_chunk_size]
+      ftp.putbinaryfile(@source, target_name, chunk_size) do |block|
         # Update counters
         transferred += block.bytesize
 
         # Update job info
         percent = (100.0 * transferred / source_size).round(1)
-        set :file_progress, percent
+        set :progress, percent
         set :file_sent, transferred
       end
 
       # Close FTP connexion
       notify "rftpd.ended"
+      set :progress, nil
       ftp.close
     end
 
+  private
+
   end
 end

data/lib/rest-ftp-daemon/logger.rb

@@ -1,7 +1,7 @@
 class Logger
   def format_message(severity, timestamp, progname, msg)
     stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
-    progname = "%-#{RestFtpDaemon::LOG_TRIM_PROGNAME}s" % progname
+    progname = "%-#{Settings[:default_trim_progname]}s" % progname
     "#{stamp}  #{severity}  #{progname}  #{msg}\n"
   end
 end

data/lib/rest-ftp-daemon/notification.rb

@@ -1,3 +1,5 @@
+require 'net/http'
+
 module RestFtpDaemon
   class Notification < RestFtpDaemon::Common
     attr_accessor :job_id

data/rest-ftp-daemon.gemspec

@@ -4,7 +4,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'rest-ftp-daemon/config'
 
 Gem::Specification.new do |spec|
-  spec.name = RestFtpDaemon::NAME
+  #spec.name = RestFtpDaemon::NAME
+  spec.name = Settings[:name]
   spec.date = Time.now.strftime("%Y-%m-%d")
   spec.authors = ["Bruno MEDICI"]
   spec.email = "rest-ftp-daemon@bmconseil.com"
@@ -16,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.version       = RestFtpDaemon::VERSION
+  spec.version       = Settings[:version]
 
   spec.required_ruby_version = '>= 1.9.3'
 
@@ -25,6 +26,8 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "thin", "~> 1.6"
   spec.add_runtime_dependency "grape"
+  spec.add_runtime_dependency "facter"
+  spec.add_runtime_dependency "settingslogic"
   spec.add_runtime_dependency "json"
 
 end

data/rest-ftp-daemon.yml.sample

@@ -0,0 +1,15 @@
+defaults: &defaults
+  daemonize: 1
+  port: 3000
+  workers: 1
+
+  transfer:
+    chunk_size: 100000
+
+  logs:
+    thin:
+    api:
+    workers:
+
+production:
+  <<: *defaults

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.41'
+  version: '0.55'
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-14 00:00:00.000000000 Z
+date: 2014-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: settingslogic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -98,8 +126,10 @@ files:
 - config.ru
 - lib/rest-ftp-daemon.rb
 - lib/rest-ftp-daemon/api/defaults.rb
+- lib/rest-ftp-daemon/api/index.haml
 - lib/rest-ftp-daemon/api/jobs.rb
 - lib/rest-ftp-daemon/api/root.rb
+- lib/rest-ftp-daemon/api/workers.rb
 - lib/rest-ftp-daemon/common.rb
 - lib/rest-ftp-daemon/config.rb
 - lib/rest-ftp-daemon/exceptions.rb
@@ -108,8 +138,8 @@ files:
 - lib/rest-ftp-daemon/logger.rb
 - lib/rest-ftp-daemon/notification.rb
 - lib/rest-ftp-daemon/worker_pool.rb
-- lib/rest-ftp-daemon/www.rb
 - rest-ftp-daemon.gemspec
+- rest-ftp-daemon.yml.sample
 - test/helper.rb
 - test/test_rest-ftp-daemon.rb
 homepage: http://github.com/bmedici/rest-ftp-daemon

data/lib/rest-ftp-daemon/www.rb

@@ -1,6 +0,0 @@
-module RestFtpDaemon
-
-  class WWW
-  end
-
-end