rest-ftp-daemon 0.241 → 0.242.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17f4eefbc675627b89a5d71dc78253b53077d0b9
-  data.tar.gz: b5df884a47a2204cc87a9ceb05ba4bfccb35a44b
+  metadata.gz: 5fe06b493b45467f188f2bb44abf1c8cc1afea28
+  data.tar.gz: dc2b03120828edc2131efba729df7faf1e042912
 SHA512:
-  metadata.gz: 54ae88ce58e703b530929940f11b87dfa30468727fcc156ae62ac88499f10d5a62ad2c3874a9b85b220ade47ee4a0d3ef4030fc4fd85a06394fd69ce3414315e
-  data.tar.gz: 74df9fd3193fe8326af484e2e4a839c63fed6ae64d8c1a4e0ec456ebba65999446424c455544365a2013af4ccecf484667cd6a6764d2b58136edb9f009909b73
+  metadata.gz: 882dbf325e495f70f89e5a948b144b6646ab4d3c1e37422d8c783fda51c7b24653e419c3b29bb7b2e0b1bc31bb5349b1896fbf30f5eabbfe86c5d81f1739c276
+  data.tar.gz: a9e48d701b53d0d7f43880d2dc401ccdfc18a8048b8b1ab38b17d4b57b7055952c69b18e5c6dbac528e630a0e0fac5379daffa4e6edfb023ecda56336e58b41f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.241)
+    rest-ftp-daemon (0.242.0)
       double-bag-ftps
       facter
       get_process_mem
@@ -41,7 +41,7 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
-    domain_name (0.5.24)
+    domain_name (0.5.25)
       unf (>= 0.0.5, < 1.0.0)
     double-bag-ftps (0.1.2)
     equalizer (0.0.11)
@@ -66,7 +66,7 @@ GEM
     haml (4.0.7)
       tilt
     hashie (3.4.2)
-    http (0.9.7)
+    http (0.9.8)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 1.0.1)
@@ -84,12 +84,12 @@ GEM
     multi_xml (0.5.5)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
-    net-ssh (2.9.2)
-    newrelic_rpm (3.13.1.300)
-    parser (2.2.2.6)
+    net-ssh (3.0.1)
+    newrelic_rpm (3.14.0.305)
+    parser (2.2.3.0)
       ast (>= 1.1, < 3.0)
     powerpack (0.1.1)
-    pry (0.10.1)
+    pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)

data/README.md

@@ -17,42 +17,45 @@ be notified of their completion, watch their status on a dedicated dashboard.
 Features
 ------------------------------------------------------------------------------------
 
-As of today, its main features are :
-
-* Offer a basic dashboard directly within the daemon HTTP interface
-* Periodically send an update-notification with transfer status and progress
-* Allow environment-specific configuration in a YAML file
-* Delegate a transfer job by `POST`'ing a simple JSON structure
-* Spawn a dedicated thread to handle this job in its own context
-* Report transfer status, progress and errors for each job in realtime
-* Expose JSON status of workers on `GET /jobs/` for automated monitoring
-* Parallelize jobs as soon as they arrive
-* Handle job queues and priority as an attribute of the job
-* Allow dynamic evaluation of priorities, and change of any attribute until the job is picked
-* Provide RESTful notifications to the requesting client
-* Allow authentication in FTP target in a standard URI-format
-* Allow configuration-based path templates to abstract local mounts or remote FTPs (endpoint tokens)
-* Allow to specify random remote/local source/target
-* Remote supported protocols: FTP and FTPs
-* Allow main file transfer protocols: sFTP, FTPs / FTPes
-* Automatically clean-up jobs after a configurable amount of time (failed, finished)
-* Current bitrate on the last blocks chunk updated in the job attributes
-* Global bitrate on the whole file transfer is re-computed after the transfer finishes
-* Daemon process is tagged with its name and environment in process lists
-* Allow basic patterns in source filename to match multiple files (`/dir/file*.jpg`)
+* System and process features
+  * environment-aware configuration in a YAML file
+  * daemon process is tagged with its name and environment in process lists
+  * global dashboard directly served within the daemon HTTP interface
+
+
+* File management ans transferts
+  * allow authentication in FTP target in a standard URI-format
+  * static path pointers in configuration to abstract local mounts or remote FTPs (endpoint tokens)
+  * local source path and local/remote target path can use patterns to match multiple files (`/dir/file*.jpg`)
+  * several file transfer protocols supported: FTPs, FTPes, sFTP
+
+* Job management
+  * highly parrallel job processing using dedicated worker threads with their own context
+  * jobs are taken into account as soon as they are submitted
+  * each job carry its own attributes: build subdirectories (mkdir), overwrite target file, priority weight
+  * dynamic evaluation of priorities, honoring any change on context until the job is picked
+  * automatically clean-up jobs after a configurable amount of time (failed, finished)
+
+* Realtime status reporting
+  * realtime transfer status reporting, with progress and errors
+  * periodic update notifications sent along with transfer status and progress to an arbitrary URL (JSON resource POSTed)
+
+
+
+Status
+------------------------------------------------------------------------------------
+
+Though it may need more robust tests, this gem has been used successfully in production for
+a while without glitches at France Télévisions.
 
 Expected features in a short-time range :
 
-* Allow fallback file source when first file path is unavailable (failover)
 * Provide swagger-style API documentation
 * Authenticate API clients
 * Allow more transfer protocols (sFTP, HTTP POST etc)
+* Expose JSON status of workers on `GET /jobs/` for automated monitoring
 
-Status
-------------------------------------------------------------------------------------
 
-Though lacking testing, this gem has been used successfully in production for
-a while without glitches.
 
 
 Installation
@@ -176,19 +179,21 @@ Default administrator credentials are `admin/admin`. Please change the password
 Logging
 ------------------------------------------------------------------------------------
 
-The application will not log to any file by default, if not specified in its configuration.
-Otherwise separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages. Providing and empty value will simply activate logging to `STDOUT`.
+The application will log to paths specified in the configuration file, if any.
+Separate logging paths can be provided for the Thin webserver, API related messages, and workers related messages.
+Providing empty values as paths, will simply activate logging to `STDOUT`.
 
 
 Job cleanup
 ------------------------------------------------------------------------------------
 
-Job can be cleanup up after a certain delay, when they are on one of these status:
+Job queue can be set to automatically cleanup after a certain delay. Entries are removed from the queue when they have been idle (updated_at) for more than X seconds, and in any of the following statuses:
 
-- "failed", cleaned up after conchita.clean_failed seconds
-- "finished", cleaned up after conchita.clean_finished seconds
+- failed (conchita.clean_failed)
+- finished (conchita.clean_finished)
+- queued, (conchita.clean_queued)
 
-Cleanup is done on a regular basis, every X seconds (X = conchita.timer)
+Cleanup is done on a regular basis, every (conchita.timer) seconds.
 
 
 TODO for this document
@@ -200,7 +205,6 @@ TODO for this document
 * Update Apiary documentation !
 * Document /status
 * Document /routes
-* Document multiple-files upload
 * Document mkdir and overwrite options
 * Document counters
 
@@ -253,6 +257,12 @@ Known bugs
 
 * As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
 
+* If you get ```fatal error: 'openssl/ssl.h' file not found when installing ```eventmachine``` on OSX El Capitan, you can try with:
+```
+gem install eventmachine -v '1.0.8' -- --with-cppflags=-I/usr/local/opt/openssl/include
+bundle install
+```
+
 
 Contributing
 ------------------------------------------------------------------------------------
@@ -293,5 +303,8 @@ About
 
 Thanks to https://github.com/berkshelf/berkshelf-api for parts and ideas used in this project
 
-Bruno MEDICI Consultant
-http://bmconseil.com/
+This project has been initiated and originally written by
+Bruno MEDICI Consultant (http://bmconseil.com/)
+
+
+

data/config.ru

@@ -9,11 +9,6 @@ $queue = RestFtpDaemon::JobQueue.new
 # Initialize workers and conchita subsystem
 $pool = RestFtpDaemon::WorkerPool.new
 
-# Rack reloader
-unless Settings.namespace == "production"
-  use Rack::Reloader, 0
-end
-
 # Rack authent
 unless Settings.adminpwd.nil?
   use Rack::Auth::Basic, "Restricted Area" do |username, password|
@@ -27,6 +22,12 @@ GC::Profiler.enable if Settings.newrelic_enabled?
 # Serve static assets
 use Rack::Static, :urls => ["/css", "/js", "/images"], :root => "#{APP_LIBS}/static/"
 
+# Rack reloader and mini-profiler
+unless Settings.namespace == "production"
+  use Rack::Reloader, 0
+  # use Rack::MiniProfiler
+end
+
 # Launch the main daemon
 run RestFtpDaemon::API::Root
 #run Rack::Cascade.new [RestFtpDaemon::API::Root]

data/lib/rest-ftp-daemon.rb

@@ -10,6 +10,14 @@ require "singleton"
 require "newrelic_rpm"
 
 
+# Development libs      /?pp=flamegraph
+# unless Settings.namespace == "production"
+#   require 'rack-mini-profiler'
+#   # require 'stackprof'
+#   require 'flamegraph'
+# end
+
+
 # Project's libs
 require_relative "rest-ftp-daemon/constants"
 require_relative "rest-ftp-daemon/settings"

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.241"
+APP_VER = "0.242.0"
 
 # Provide default config file information
 APP_LIB = File.expand_path File.dirname(__FILE__)
@@ -91,7 +91,8 @@ DASHBOARD_WORKER_STYLES = {
 DEFAULT_WORKER_TIMEOUT  = 3600
 DEFAULT_FTP_CHUNK       = 1024
 DEFAULT_PAGE_SIZE       = 40
-DEFAULT_WORKERS         = 1
+DEFAULT_WORKERS         = 2
+DEFAULT_RETRY_DELAY     = 10
 
 DEFAULT_SFTP_TIMEOUT    = 30
 

data/lib/rest-ftp-daemon/paginate.rb

@@ -1,6 +1,8 @@
 module RestFtpDaemon
   class Paginate
 
+    attr_writer :only
+
     def initialize data
       # Defaults
       @pages = 0
@@ -22,10 +24,6 @@ module RestFtpDaemon
       @pages = 1 if @pages < 1
     end
 
-    def only= raw_only
-      @only = raw_only
-    end
-
     def page= raw_page
       @page = [1, raw_page.to_i, @pages].sort[1]
     end

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -13,14 +13,9 @@
 
   %tr{class: trclass.to_s}
     %td
-
-
       %a{href: Helpers.dashboard_job_link(job)}
         %b= job.id
 
-
-
-
     %td= job.label
 
     %td{title: job.get(:source_path)}
@@ -78,8 +73,14 @@
         .label.label-default.flag.worker-label= job.priority
 
     %td
-      - unless job.runs.nil? || job.runs.zero?
-        .label.label-info.flag.worker-label= job.runs
+      - if job.runs.nil?
+        .label.label-info.flag.worker-label= "-"
+      - elsif job.runs == 1
+        .label.label-default.flag.worker-label= job.runs
+      - elsif job.runs == 2
+        .label.label-warning.flag.worker-label= job.runs
+      - elsif job.runs > 2
+        .label.label-danger.flag.worker-label= job.runs
 
     -#%td
       - unless job.priority.nil?

data/lib/rest-ftp-daemon/worker_job.rb

@@ -31,6 +31,7 @@ module RestFtpDaemon
       on_errors = Settings.at(:retry, :on_errors)
       max_age = Settings.at(:retry, :max_age)
       max_runs = Settings.at(:retry, :max_runs)
+      delay = Settings.at(:retry, :delay)
 
       if !job.error
         #log_info "job succeeded"
@@ -39,13 +40,21 @@ module RestFtpDaemon
         log_error "not retrying: error not eligible"
 
       elsif max_age && (job.age >= max_age)
-        log_error "not retrying: too old (max_age: #{max_age})"
+        log_error "not retrying: max_age reached (#{max_age} s)"
 
       elsif max_runs && (job.runs >= max_runs)
-        log_error "not retrying: too many runs (max_runs: #{max_runs})"
+        log_error "not retrying: max_runs reached (#{max_runs} tentatives)"
 
       else
-        log_info "retrying job: requeued"
+        # Delay cannot be negative, and will be 1s minimum
+        retry_after = [delay || DEFAULT_RETRY_DELAY, 1].max
+        log_info "retrying job: waiting for #{retry_after} seconds"
+
+        # Wait !
+        sleep retry_after
+        log_info "retrying job: requeued after delay"
+
+        # Now, requeue this job
         $queue.requeue job
       end
 

data/rest-ftp-daemon.gemspec

@@ -32,6 +32,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "http", "~> 0.8"
   spec.add_development_dependency "rubocop", "~> 0.32.0"
   spec.add_development_dependency "pry"
+  # spec.add_development_dependency "rack-mini-profiler"
+  # spec.add_development_dependency "stackprof"
+  # spec.add_development_dependency "flamegraph"
 
   spec.add_runtime_dependency "thin", "~> 1.6"
   spec.add_runtime_dependency "grape"

data/rest-ftp-daemon.yml.sample

@@ -17,6 +17,7 @@ defaults: &defaults
     # on_errors: ["net_temp_error", "conn_reset_by_peer", "conn_timed_out", "conn_refused", "sftp_auth_failed", "conn_host_is_down", "conn_unreachable", "conn_failed", "conn_openssl_error"]
     # max_runs: 5
     # max_age: 1800
+    # delay: 10
 
   conchita:
     # timer: 60

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.241'
+  version: 0.242.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-24 00:00:00.000000000 Z
+date: 2015-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler