rest-ftp-daemon 0.100 → 0.100.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1cd7cf3fc402f4257b9c878b5227467f3f250c95
-  data.tar.gz: 441d0c886af8f6ba67347a82318efb40521862ce
+  metadata.gz: a45abd6abf0d38a6dca977c42e81c1cf538dbd77
+  data.tar.gz: a0d3f0fd02e570f9f0f3d56ae403607e0d854a43
 SHA512:
-  metadata.gz: e774def2fa7622368cca0d28915aaf6c45a121c438d6a4da34c408e733fc79c5611a0c72b1355c1200d5c49ec00d52b8a70198f243442d607e65f8b85ebbd7f7
-  data.tar.gz: 6696902d94a1130e840955a84c4cde5fac53b0df16bdebeb581c27ff7730efc12b22bf984a580be37448ea35e8aded345798ddc918fe1b89e7f90af7c0615902
+  metadata.gz: f3d570450921acd5a1c8d0c61c89ad6ce6b0a97be868e74f3cacf654b3a7695e055345aa3111ca74f068572786b9d52fdbcaa1c2315efee26f8c924211890707
+  data.tar.gz: 733164d4039e1660d50850b889c17704718a65097337d9a9b368ea1b3fc29879e67d4ef27e597a102b8775a1c58ec4dc8461bb81d8be2c790b393bce31d02ca3

data/README.md

@@ -13,6 +13,8 @@ Features
 
 As of today, its main features are :
 
+* Offer a basic dashboard directly within the daemon HTTP interface
+* Periodically send an update-notification with transfer status and progress
 * Allow environment-specific configuration in a YAML file
 * Delegate a transfer job by ``POST```'ing a simple JSON structure
 * Spawn a dedicated thread to handle this job in its own context
@@ -24,6 +26,7 @@ As of today, its main features are :
 * Provide RESTful notifications to the requesting client
 * Allow authentication in FTP target in a standard URI-format
 * Allow configuration-based path templates to abstract local mounts or remote FTPs (endpoint tokens)
+* Allow to specify random remote/local source/target
 * Remote supported protocols: FTP and FTPs
 * Allow main file transfer protocols: sFTP, FTPs / FTPes
 * Automatically clean-up jobs after a configurable amount of time (failed, finished)
@@ -34,13 +37,9 @@ As of today, its main features are :
 
 Expected features in a short-time range :
 
-* Allow change of priorities or other attributes after a job has been started
-* Offer a basic dashboard directly within the daemon HTTP interface
-* Periodically send an update-notification with transfer status and progress
 * Allow fallback file source when first file path is unavailable (failover)
 * Provide swagger-style API documentation
 * Authenticate API clients
-* Allow to specify random remote/local source/target
 * Allow more transfer protocols (sFTP, HTTP POST etc)
 
 Known bugs :
@@ -58,8 +57,8 @@ You may use ```rbenv``` and ```ruby-build``` to get the right Ruby version. If t
 ```
 # apt-get install ruby-build rbenv
 # ruby-build --definitions | grep '2.1'
-
 ```
+
 Otherwise, you way have to update ruby-build to include Ruby 2.1.0 definitions.
 On Debian, 2.1.0 is not included in Wheezy and appears in Jessie's version of the package.
 
@@ -90,18 +89,6 @@ Update RubyGems and install the gem from rubygems.org
 # rest-ftp-daemon start
 ```
 
-Finally start the daemon on the standart port, or on a specific port using ```-p```
-
-```
-# rest-ftp-daemon -p 4000 start
-```
-
-Check that the daemon is running and providing its status info.
-If the daemon seems to exit as soon as it's launched, this may be due to logfiles that cannot be written on (check permissions or owner).
-
-```
-http://localhost:3200/
-```
 
 Configuration
 ------------------------------------------------------------------------------------
@@ -123,6 +110,38 @@ As a starting point, ``rest-ftp-daemon.yml.sample`` is an exemple config file th
 Default administrator credentials are admin/admin. Please change the password in this configuration file before starting any kind of production.
 
 
+Execution
+------------------------------------------------------------------------------------
+
+You can simply start the daemon on the standart port, or on a specific port using ```-p```
+
+```
+# rest-ftp-daemon -p 4000 start
+```
+
+Check that the daemon is running and exposes a JSON status structure on ```http://localhost:3200/status```.
+
+The dashbaord will provide a gobal view on ```http://localhost:3200/```
+
+If the daemon appears to exit quickly when launched, it may be caused by logfiles that can't be written (check files permissions or owner).
+
+Launcher options :
+
+| Param   | Short         | Default       | Description                                                 |
+|-------  |-------------- |-------------  |-----------------------------------------------------------  |
+| -p      | --port        | (automatic)   | Port to listen for API requests                             |
+| -e      |               | production    | Environment name                                            |
+|         | --dev         |               | Equivalent to -e development                                |
+| -w      | --workers     | 1             | Number of workers spawned at launch                         |
+| -d      | --daemonize   | false         | Wether to send the daemon to background                     |
+| -f      | --foreground  | false         | Wether to keep the daemon running in the shell              |
+| -P      | --pid         | (automatic)   | Path of the file containing the PID                         |
+| -u      | --user        | (none)        | User to run the daemon as                                   |
+| -g      | --group       | (none)        | Group of the user to run the daemon as                      |
+| -h      | --help        |               | Show info about the current version and available options   |
+| -v      | --version     |               | Show the current version                                    |
+
+
 Logging
 ------------------------------------------------------------------------------------
 
@@ -133,12 +152,12 @@ Otherwise separate logging paths can be provided for the Thin webserver, API rel
 Job cleanup
 ------------------------------------------------------------------------------------
 
-Job can be cleanup up after a certain amount of time, when they reach on of these status:
+Job can be cleanup up after a certain delay, when they are on one of these status:
 
-- failed, after conchita.clean_failed seconds
-- finished, after conchita.clean_finished seconds
+- "failed", cleaned up after conchita.clean_failed seconds
+- "finished", cleaned up after conchita.clean_finished seconds
 
-Cleanup is done on a regular basis, every few seconds (conchita.timer)
+Cleanup is done on a regular basis, every X seconds (X = conchita.timer)
 
 
 Usage examples
@@ -184,20 +203,18 @@ GET http://localhost:3100/jobs/1
 ```
 
 
-
-Doc TODO
+TODO for this document
 ------------------------------------------------------------------------------------
 
-Document /status
-
-Document /routes
-
-Document multiple-files upload
-
-Document mkdir and overwrite options
-
-Document counters
-
+* Update Apiary documentation
+* Update Apiary documentation
+* Update Apiary documentation
+* Update Apiary documentation !
+* Document /status
+* Document /routes
+* Document multiple-files upload
+* Document mkdir and overwrite options
+* Document counters
 
 
 About

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -17,10 +17,14 @@ module RestFtpDaemon
 
         expose :error
         expose :status
-
         expose :queued_at
         expose :updated_at
+        expose :started_at
+        expose :finished_at
+
+        # Computed fields
         expose :age
+        expose :exectime
 
         # Params
         # expose :wid, unless: lambda { |object, options| object.wid.nil? }

data/lib/rest-ftp-daemon/config.rb

@@ -18,4 +18,3 @@ class Settings < Settingslogic
   end
 
 end
-

data/lib/rest-ftp-daemon/constants.rb

@@ -1,21 +1,20 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_CONF = "/etc/#{APP_NAME}.yml"
-APP_VER = "0.100"
+APP_VER = "0.100.2"
 
 # Some global constants
 IDENT_JOB_LEN = 4
 IDENT_NOTIF_LEN = 4
 IDENT_RANDOM_LEN = 8
+DEFAULT_LOGS_PIPE_LEN = 15
+DEFAULT_LOGS_ID_LEN = 8
 
 # Some defaults
 DEFAULT_CONNECT_TIMEOUT_SEC = 30
 DEFAULT_UPDATE_EVERY_KB = 2048
 DEFAULT_WORKERS = 1
 
-DEFAULT_LOGS_PIPE_WIDTH = 15
-DEFAULT_LOGS_ID_WIDTH = 8
-
 # Initialize markers
 APP_STARTED = Time.now
 APP_LIBS = File.dirname(__FILE__)

data/lib/rest-ftp-daemon/job.rb

@@ -17,6 +17,9 @@ module RestFtpDaemon
     attr_reader :queued_at
     attr_reader :updated_at
 
+    attr_reader :started_at
+    attr_reader :finished_at
+
     attr_reader :params
 
     FIELDS.each do |field|
@@ -34,6 +37,12 @@ module RestFtpDaemon
         instance_variable_set("@#{field.to_s}", params[field])
       end
       @params = {}
+      @updated_at = nil
+      @started_at = nil
+      @finished_at = nil
+      @error = nil
+      @status = nil
+      @wid = nil
 
       # Logger
       # @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
@@ -173,10 +182,15 @@ module RestFtpDaemon
   protected
 
     def age
-      return 0 if @queued_at.nil?
+      return nil if @queued_at.nil?
       (Time.now - @queued_at).round(2)
     end
 
+    def exectime
+      return nil if (@started_at.nil? || @finished_at.nil?)
+      (@finished_at - @started_at).round(2)
+    end
+
     def wander time
       info "Job.wander #{time}"
       @wander_for = time
@@ -267,6 +281,9 @@ module RestFtpDaemon
     end
 
     def transfer
+      # Start transfert flag
+      @started_at = Time.now
+
       # Method assertions and init
       @status = :checking_source
       raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
@@ -311,13 +328,14 @@ module RestFtpDaemon
         set :source_processed, done
       end
 
-      # Add total transferred to counter
-      $queue.counter_add :transferred, @transfer_total
-
       # Close FTP connexion
       @ftp.close
       info "Job.transfer disconnecting"
       @status = :disconnecting
+
+      # Update counters and flags
+      $queue.counter_add :transferred, @transfer_total
+      @finished_at = Time.now
     end
 
   private
@@ -524,7 +542,7 @@ module RestFtpDaemon
         stack << (Helpers.format_bytes @transfer_sent, "B")
         stack << (Helpers.format_bytes @transfer_total, "B")
         stack << (Helpers.format_bytes bitrate0, "bps")
-        info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PIPE_WIDTH.to_i}s" % txt)}.join("\t")
+        info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PIPE_LEN.to_i}s" % txt)}.join("\t")
 
         # Update time pointer
         t0 = Time.now

data/lib/rest-ftp-daemon/job_queue.rb

@@ -67,12 +67,14 @@ module RestFtpDaemon
     end
 
     def counter_get name
-      @counters[name]
+      @mutex_counters.synchronize do
+        @counters[name]
+      end
     end
 
     def counters
       @mutex_counters.synchronize do
-        @counters.clone
+        @counters
       end
     end
 
@@ -90,8 +92,6 @@ module RestFtpDaemon
     end
 
     def all
-      # queued2 = @queued.clone
-      # return queued2.merge(@popped)
       @queued + @popped
     end
     def all_size
@@ -115,7 +115,6 @@ module RestFtpDaemon
       @mutex.synchronize do
         # Push job into the queue
         @queued.push job
-        #info "JobQueue.push: #{job.id}"
 
         # Tell the job it's been queued
         job.set_queued if job.respond_to? :set_queued
@@ -132,7 +131,6 @@ module RestFtpDaemon
     alias << push
     alias enq push
 
-
     def pop(non_block=false)
       # info "JobQueue.pop"
       @mutex.synchronize do
@@ -165,11 +163,15 @@ module RestFtpDaemon
     end
 
     def ordered_queue
-      @queued.sort_by { |item| [item.priority.to_i, - item.id.to_i] }
+      @mutex_counters.synchronize do
+        @queued.sort_by { |item| [item.priority.to_i, - item.id.to_i] }
+      end
     end
 
     def ordered_popped
-      @popped.sort_by { |item| [item.updated_at] }
+      @mutex_counters.synchronize do
+        @popped.sort_by { |item| [item.updated_at] }
+      end
     end
 
   protected

data/lib/rest-ftp-daemon/logger.rb

@@ -3,7 +3,7 @@ class Logger
   attr_accessor :pipe
 
   def info_with_id message, options = {}
-    field_id = "%#{-DEFAULT_LOGS_ID_WIDTH.to_i}s" % options[:id].to_s
+    field_id = "%#{-DEFAULT_LOGS_ID_LEN.to_i}s" % options[:id].to_s
     add Logger::INFO, "#{field_id} \t#{'  '*(options[:level].to_i+1)}#{message}"
   end
 

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -25,7 +25,7 @@ module RestFtpDaemon
       logger.formatter = proc do |severity, datetime, progname, message|
         # stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
         stamp = datetime.strftime("%Y-%m-%d %H:%M:%S")
-        field_pipe = "%-#{DEFAULT_LOGS_PIPE_WIDTH.to_i}s" % progname
+        field_pipe = "%-#{DEFAULT_LOGS_PIPE_LEN.to_i}s" % progname
         "#{stamp} #{field_pipe} #{message}\n"
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.100'
+  version: 0.100.2
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-10 00:00:00.000000000 Z
+date: 2014-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler