rest-ftp-daemon 0.435.2 → 0.501.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 33e255f828fa170324200b0f55874d10e6855e07
-  data.tar.gz: e3cb2d11b564f572293286e1afd083affb4da4c4
+  metadata.gz: 180577d1189952b25cbcecf2a7ba1144116d2c41
+  data.tar.gz: 013a7a6dbaac918ef25e2d3607e9d7cc89efb45e
 SHA512:
-  metadata.gz: ffd7d9b083f8787d12cf50c15faeb3f479cc552ab2fd8181a05db9405cb90bf9a52411b39ce5af80e7eac02ddfe6a20f37c6e65265bcb2172af12fdd1f7f7795
-  data.tar.gz: a8dacb6c52cefacd1954b92709938fef737880dcdf4e3e9c346ff6c158e1e7c2f1ccea39ae02e4823993c58f625ff972e5c8fb7d6d626da3fcfb49e1f75cc055
+  metadata.gz: c0328a020bf3a3c59d649be271d92e3a2a6661ae0e654255e323c2fb5e6a3e18fbc425c48a905a8c1b84971c1251a393e51135c049d2b3192b2fa2f23307bcb7
+  data.tar.gz: 3b42ec7c4feef47bb5005b338a56f4bb45e094919d97b00f822a66987cc7f749d65ad4912a42384bf7bece3acba7dc187a68ec184b3b976ec29b6c0a1273972b

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.435.2)
+    rest-ftp-daemon (0.501.0)
       activesupport (~> 4.2)
       api-auth
       aws-sdk-resources (~> 2)
-      bmc-daemon-lib (~> 0.4.3)
+      bmc-daemon-lib (~> 0.5.0)
       double-bag-ftps
       facter
       get_process_mem
@@ -34,18 +34,21 @@ GEM
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    addressable (2.4.0)
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
     api-auth (2.0.1)
     ast (2.3.0)
-    aws-sdk-core (2.6.2)
+    aws-sdk-core (2.6.26)
+      aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.2)
-      aws-sdk-core (= 2.6.2)
+    aws-sdk-resources (2.6.26)
+      aws-sdk-core (= 2.6.26)
+    aws-sigv4 (1.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    bmc-daemon-lib (0.4.3)
+    bmc-daemon-lib (0.5.1)
       chamber (~> 2.9.1)
     builder (3.2.2)
     chamber (2.9.1)
@@ -60,30 +63,29 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
-    domain_name (0.5.20160826)
+    domain_name (0.5.20161021)
       unf (>= 0.0.5, < 1.0.0)
     double-bag-ftps (0.1.3)
-    enumerable-lazy (0.0.1)
     equalizer (0.0.11)
-    eventmachine (1.2.0.1)
+    eventmachine (1.2.1)
     facter (2.4.6)
       CFPropertyList (~> 2.2.6)
     ffi (1.9.14)
     get_process_mem (0.2.1)
-    grape (0.17.0)
+    grape (0.18.0)
       activesupport
       builder
       hashie (>= 2.1.0)
       multi_json (>= 1.3.2)
       multi_xml (>= 0.5.2)
-      mustermann19 (~> 0.4.3)
+      mustermann-grape (~> 0.4.0)
       rack (>= 1.3.0)
       rack-accept
       virtus (>= 1.0.0)
-    grape-entity (0.5.1)
+    grape-entity (0.6.0)
       activesupport
       multi_json (>= 1.3.2)
-    grape-swagger (0.23.0)
+    grape-swagger (0.25.0)
       grape (>= 0.12.0)
     grape-swagger-entity (0.1.4)
       grape-entity
@@ -94,12 +96,12 @@ GEM
     haml (4.0.7)
       tilt
     hashie (3.4.6)
-    http (2.0.3)
+    http (2.1.0)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 1.0.1)
       http_parser.rb (~> 0.6.0)
-    http-cookie (1.0.2)
+    http-cookie (1.0.3)
       domain_name (~> 0.5)
     http-form_data (1.0.1)
     http_parser.rb (0.6.0)
@@ -109,42 +111,45 @@ GEM
     json (1.8.3)
     method_source (0.8.2)
     mime-types (2.99.3)
-    minitest (5.9.0)
+    minitest (5.9.1)
     multi_json (1.12.1)
     multi_xml (0.5.5)
-    mustermann19 (0.4.4)
-      enumerable-lazy
+    mustermann (0.4.0)
+      tool (~> 0.2)
+    mustermann-grape (0.4.0)
+      mustermann (= 0.4.0)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
     net-ssh (3.2.0)
     netrc (0.11.0)
-    newrelic_rpm (3.16.2.321)
-    parser (2.3.1.4)
+    newrelic_rpm (3.17.1.326)
+    parser (2.3.2.0)
       ast (~> 2.2)
     powerpack (0.1.1)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
+    public_suffix (2.0.4)
     rack (2.0.1)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rainbow (2.1.0)
     rake (11.3.0)
-    representable (3.0.0)
+    representable (3.0.1)
       declarative (~> 0.0.5)
-      uber (~> 0.0.15)
+      uber (>= 0.0.15, < 0.2.0)
     rest-client (1.8.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
-    rollbar (2.13.1)
+    rollbar (2.13.3)
       multi_json
     rspec (3.5.0)
       rspec-core (~> 3.5.0)
       rspec-expectations (~> 3.5.0)
       rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.3)
+    rspec-core (3.5.4)
       rspec-support (~> 3.5.0)
     rspec-expectations (3.5.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -153,7 +158,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.5.0)
     rspec-support (3.5.0)
-    rubocop (0.43.0)
+    rubocop (0.45.0)
       parser (>= 2.3.1.1, < 3.0)
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
@@ -162,7 +167,7 @@ GEM
     ruby-prof (0.16.2)
     ruby-progressbar (1.8.1)
     slop (3.6.0)
-    streamio-ffmpeg (3.0.0)
+    streamio-ffmpeg (3.0.2)
       multi_json (~> 1.8)
     sys-cpu (0.7.2)
       ffi
@@ -173,9 +178,10 @@ GEM
     thor (0.19.1)
     thread_safe (0.3.5)
     tilt (2.0.5)
+    tool (0.2.3)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    uber (0.0.15)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.2)
@@ -200,4 +206,4 @@ DEPENDENCIES
   ruby-prof
 
 BUNDLED WITH
-   1.12.5
+   1.13.5

data/config.ru

@@ -30,7 +30,7 @@ use Rack::Static, root: "#{Conf.app_libs}/static/", urls: [
 case Conf.app_env
   when "development"
     # Rack reloader
-    # use Rack::Reloader, 1
+    use Rack::Reloader, 1
 
     # Newrelic dev mode
     require 'new_relic/rack/developer_mode'

data/defaults.yml

@@ -5,6 +5,11 @@ user: rftpd
 # host: "myhost"
 allow_reload: false
 
+# FUTURE
+debug:
+    ftp: false
+    location: false
+
 pools:                      # number of workers decidated to each pool value
   default: 2
   urgent: 1
@@ -28,7 +33,6 @@ transfer:
   overwrite: false          # overwrite any target file with the same name
   timeout: 1800             # jobs running for longer than X seconds will be killed
   notify_after: 5           # wait at least X seconds between HTTP notifications
-
   debug_s3: false
   debug_ftp: false
   debug_ftps: false
@@ -68,16 +72,16 @@ logs:
   path:     "/tmp/"
   level:    debug
 
-  thin:     "rftpd-default-thin.log"
-  default:  "rftpd-default-core.log"
+  thin:     "rftpd-default.log"
+  default:  "rftpd-default.log"
 
-  queue:    "rftpd-default-core.log"
-  api:      "rftpd-default-core.log"
-  workers:  "rftpd-default-core.log"
-  transfer: "rftpd-default-workers.log"
-  conchita: "rftpd-default-workers.log"
-  reporter: "rftpd-default-workers.log"
-  notify:   "rftpd-default-workers.log"
+  queue:    "rftpd-default.log"
+  api:      "rftpd-default.log"
+  workers:  "rftpd-default.log"
+  transfer: "rftpd-default.log"
+  conchita: "rftpd-default.log"
+  reporter: "rftpd-default.log"
+  notify:   "rftpd-default.log"
 
-  newrelic: "rftpd-default-newrelic.log"
-  rollbar:  "rftpd-default-rollbar.log"
+  newrelic: "rftpd-newrelic.log"
+  rollbar:  "rftpd-rollbar.log"

data/lib/rest-ftp-daemon.rb

@@ -13,6 +13,7 @@ require "rollbar"
 # Shared libs / monkey-patching
 require_relative "shared/patch_array"
 require_relative "shared/patch_haml"
+require_relative "shared/patch_file"
 
 # Project's libs
 require_relative "rest-ftp-daemon/constants"

data/lib/rest-ftp-daemon/api/config.rb

@@ -11,7 +11,6 @@ module RestFtpDaemon
         end
       end
 
-
       ### ENDPOINTS
       desc "Show daemon config"
       get "/" do
@@ -33,4 +32,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -8,6 +8,8 @@ module RestFtpDaemon
   module API
     class Dashboard < Grape::API
       include BmcDaemonLib
+      content_type :html, 'application/html'
+      format :html
 
       ### HELPERS
       helpers do
@@ -61,10 +63,10 @@ module RestFtpDaemon
           output = render :dashboard
 
           # Send response
-          env["api.format"] = :html
-          format "html"
+          #env["api.format"] = :html
+          # format "html"
           status 200
-          content_type "text/html"
+          content_type "html"
           body output
         end
 
@@ -85,4 +87,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/api/debug.rb

@@ -18,7 +18,7 @@ module RestFtpDaemon
         def debug_encodings
           # Encodings
           encodings = {}
-          jobs = RestFtpDaemon::JobQueue.instance.jobs
+          jobs = JobQueue.instance.jobs
 
           jobs.each do |job|
             # here = out[job.id] = {}
@@ -27,7 +27,7 @@ module RestFtpDaemon
             me[:error] = job.error.encoding.to_s unless job.error.nil?
             me[:status] = job.status.encoding.to_s unless job.status.nil?
 
-            RestFtpDaemon::Job::IMPORTED.each do |name|
+            Job::IMPORTED.each do |name|
               value = job.send(name)
               me[name] = value.encoding.to_s if value.is_a? String
             end
@@ -46,11 +46,11 @@ module RestFtpDaemon
 
         # Extract routes
         routes = []
-        RestFtpDaemon::API::Root.routes.each do |route|
+        API::Root.routes.each do |route|
           routes << {
             url: "#{route.options[:method]} #{route.pattern.path}",
             vars: route.instance_variables,
-            options: route.options
+            options: route.options,
             }
 
         end
@@ -60,9 +60,10 @@ module RestFtpDaemon
           metrics: debug_metrics,
           encodings: debug_encodings,
           routes: routes,
+          content_types: Grape::ContentTypes::CONTENT_TYPES
           }
       end
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -171,4 +171,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/api/root.rb

@@ -31,7 +31,8 @@ module RestFtpDaemon
 
         def exception_error error, http_code, exception
           # Extract message lines
-          lines = exception.message.lines.collect(&:strip).reject(&:empty?)
+          lines = exception.message.lines
+            #.lines.collect(&:strip).reject(&:empty?)
 
           # Log error to file
           log_error "[#{error}] [#{http_code}] #{lines.shift} ", lines
@@ -67,9 +68,9 @@ module RestFtpDaemon
       # version 'v1'
 
       # Response formats
-      content_type :json, 'application/json; charset=utf-8'
-      # format :json
-      default_format :json
+      #content_type :json, 'application/json; charset=utf-8'
+      format :json
+      # default_format :json
 
       # Pretty JSON
       # formatter :json_tmp, ->(object, env) do
@@ -78,11 +79,11 @@ module RestFtpDaemon
       # end
 
       ### MOUNTPOINTS
-      mount RestFtpDaemon::API::Status      => MOUNT_STATUS
-      mount RestFtpDaemon::API::Jobs        => MOUNT_JOBS
-      mount RestFtpDaemon::API::Dashboard   => MOUNT_BOARD
-      mount RestFtpDaemon::API::Config      => MOUNT_CONFIG
-      mount RestFtpDaemon::API::Debug       => MOUNT_DEBUG
+      mount API::Status      => MOUNT_STATUS
+      mount API::Jobs        => MOUNT_JOBS
+      mount API::Dashboard   => MOUNT_BOARD
+      mount API::Config      => MOUNT_CONFIG
+      mount API::Debug       => MOUNT_DEBUG
 
 
       ### API Documentation
@@ -109,4 +110,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/api/status.rb

@@ -49,4 +49,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/constants.rb

@@ -3,17 +3,25 @@ DEFAULT_POOL            = "default"
 DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
 DEFAULT_PAGE_SIZE       = 50    # 50 lines
 DEFAULT_RETRY_AFTER     = 10    # 10s
-TARGET_BLANK             = "_blank"
+TARGET_BLANK            = "_blank"
+KB                      = 1024
+MB                      = 1024*KB
+GB                      = 1024*MB
+
 
 # Internal job constants
 JOB_RANDOM_LEN          = 8
 JOB_IDENT_LEN           = 4
 JOB_TEMPFILE_LEN        = 8
-JOB_FTP_CHUNKMB         = 2048   # 2 MB
+
+JOB_FTP_CHUNKMB         = 2*MB
+JOB_S3_MIN_PART         = 5*MB
+JOB_S3_MAX_COUNT        = 10_000
 
 JOB_FFMPEG_THREADS      = 2
 JOB_FFMPEG_ATTRIBUTES   = [:video_codec, :video_bitrate, :video_bitrate_tolerance, :frame_rate, :resolution, :aspect, :keyframe_interval, :x264_vprofile, :x264_preset, :audio_codec, :audio_bitrate, :audio_sample_rate, :audio_channels]
 
+
 # Internal job infos
 INFO_PARAMS               = :params
 INFO_ERROR_MESSAGE        = :error_message
@@ -34,10 +42,10 @@ LOGGER_FORMAT = {
   # context:  "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s ",
   # context:  "wid:%-8{wid} jid:%-12{jid} id:%-5{id}",
   context: {
-    caller: "%-17s",
     wid:    "%-10s",
     jid:    "%-10s",
     id:     "%-8s",
+    caller: "%18s |",
     }
   }
 
@@ -123,3 +131,4 @@ MOUNT_CONFIG             = "/config"
 # Notifications
 NOTIFY_PREFIX           = "rftpd"
 NOTIFY_IDENTIFIER_LEN   = 4
+