rest-ftp-daemon 0.423.3 → 0.424.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2802c64bdd28d6bd4e0101b91bc66899c0e4b3c
-  data.tar.gz: a1c767f0d1238a66424626d2923711dab7ba78e4
+  metadata.gz: 2d4d6f823c82eb53007aceccb525111fce662019
+  data.tar.gz: b779d0afa8de9f3f4cf3e935574223f24883e331
 SHA512:
-  metadata.gz: f995227ffb577e2b80946cdbbd3c0fa27b275ee855daada08502e0ee39c471a12a62c98ac3cf77afcb5ed07d8313dede12f29a6a7a6d66f8742041a14b10e64d
-  data.tar.gz: 8307a7e971a9a24bc40f59a257258ef9b700131918170cb208129a42e626eb9e3969b3e6a796224ec5809dccc2391719694eedb49550badc44212bdf3b6c0a0d
+  metadata.gz: 67fafa5d1998d4d1a7f9b28fdbfb20b6637e2f506eea4799a277caf3d326133d1e56a554991de05329fc44875268e5a6fb43283ec92c595bb50b98de205b166c
+  data.tar.gz: d0dbc285d27a26120887795803c35f250f415ecc1b9d8fd8a1937be313c1767f4b18d31e987a82c944c7374ea21f67cc218f01b7198c74edf344fc4cd3e60b03

data/.codeclimate.yml

@@ -0,0 +1,30 @@
+engines:
+  rubocop:
+    enabled: true
+    #checks:
+    # Rubocop/Metrics/ClassLength:
+    #   enabled: false
+  brakeman:
+    enabled: true
+    #exclude_fingerprints:
+    #- 08c185b3f764f262539a1f76997d3a89
+  eslint:
+    enabled: true
+  csslint:
+    enabled: true
+  duplication:
+    enabled: true
+    config:
+      languages:
+      - ruby
+        #mass_threshold: 30
+      - javascript
+ratings:
+  paths:
+  - app/**
+  - lib/**
+  - "**.rb"
+  - "**.go"
+exclude_paths:
+- spec/**/*
+- "**/swagger/**/*"

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.423.3)
+    rest-ftp-daemon (0.424.0)
       activesupport (~> 4.2)
       api-auth
       aws-sdk-resources (~> 2)
-      bmc-daemon-lib (~> 0.3.4)
+      bmc-daemon-lib (~> 0.3.8)
       double-bag-ftps
       facter
       get_process_mem
@@ -19,7 +19,7 @@ PATH
       net-sftp
       newrelic_rpm
       rest-client (~> 1.8)
-      settingslogic
+      rollbar
       streamio-ffmpeg
       sys-cpu
       thin (~> 1.7)
@@ -39,15 +39,15 @@ GEM
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    aws-sdk-core (2.5.8)
+    aws-sdk-core (2.5.10)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.5.8)
-      aws-sdk-core (= 2.5.8)
+    aws-sdk-resources (2.5.10)
+      aws-sdk-core (= 2.5.10)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    bmc-daemon-lib (0.3.4)
+    bmc-daemon-lib (0.3.8)
       chamber (~> 2.9)
     builder (3.2.2)
     chamber (2.9.1)
@@ -140,6 +140,8 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
+    rollbar (2.12.0)
+      multi_json
     rspec (3.5.0)
       rspec-core (~> 3.5.0)
       rspec-expectations (~> 3.5.0)
@@ -160,7 +162,6 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.4)
     ruby-progressbar (1.8.1)
-    settingslogic (2.0.9)
     slop (3.6.0)
     streamio-ffmpeg (2.1.0)
       multi_json (~> 1.8)

data/README.md

@@ -53,7 +53,7 @@ a while without any glitches at France Télévisions.
 
 #### API Documentation
 
-API documentation was [maintained on Apiary](http://docs.restftpdaemon.apiary.io/) and will me migrated to swagger on day or another.
+API documentation is self-hosted on ```/swagger.html```
 
 #### Expected features in a short-time range
 
@@ -279,10 +279,6 @@ logs:
 TODO for this document
 ------------------------------------------------------------------------------------
 
-* Update Apiary documentation
-* Update Apiary documentation
-* Update Apiary documentation
-* Update Apiary documentation !
 * Document /status
 * Document /routes
 * Document mkdir and overwrite options

data/bin/rest-ftp-daemon

@@ -24,6 +24,7 @@ begin
   cmd_user = nil
   cmd_group = nil
   cmd_daemonize = nil
+  cmd_dump = nil
 
   # Init Chamber-based configuration from Gemspec
   Conf.init File.dirname(__FILE__) + "/../"
@@ -31,9 +32,10 @@ begin
   # Parse options and check compliance
   parser = OptionParser.new do |opts|
     opts.banner = "Usage: #{File.basename $PROGRAM_NAME} [options] start|stop"
-    opts.on("-c", "--config CONFIGFILE")     { |value| cmd_config = File.expand_path(value.to_s)}
-    opts.on("-e", "--environment ENV")       { |value| Conf.app_env = value }
-    opts.on("",   "--dev")                   { Conf.app_env = "development" }
+    opts.on("-c", "--config CONFIGFILE")                                  { |value| cmd_config = File.expand_path(value.to_s)}
+    opts.on("",   "--dump", "Dump config as seen by the process")         { |value| cmd_dump = true }
+    opts.on("-e", "--environment ENV")                                    { |value| Conf.app_env = value }
+    opts.on("",   "--dev")                                                { Conf.app_env = "development" }
     opts.on("-p", "--port PORT", "use PORT")                              { |value| cmd_port = value.to_i }
     opts.on("-d", "--daemonize", "Run daemonized in the background")      { cmd_daemonize = true }
     opts.on("-f", "--foreground", "Run in the foreground")                { cmd_daemonize = false }
@@ -120,10 +122,14 @@ puts
 puts "Daemonize       \t #{Conf[:daemonize]}"
 puts "Network port    \t #{Conf[:port]}"
 puts "User:group      \t #{Conf[:user]}:#{Conf[:group]}" if Conf[:user] && Conf[:group]
-puts "Newrelic        \t #{Conf.newrelic_enabled? ? Conf.at(:newrelic, :license) : 'no'}"
 puts "PID file        \t #{Conf[:pidfile]}"
 puts
-puts Conf.dump
+puts "Newrelic        \t #{Conf.feature?(:newrelic) || '-'}"
+puts "Rollbar         \t #{Conf.feature?(:rollbar) || '-'}"
+if cmd_dump
+  puts
+  puts Conf.dump
+end
 puts
 puts "--- Thin ARGV"
 puts argv.flatten.join(" ")

data/defaults.yml

@@ -56,18 +56,24 @@ newrelic:
   prefix: "rftpd"                         # app prefix to build app_name
   # platform: "bigbusiness"               # app platform to build app_name
 
+rollbar:
+  debug: false
+  # token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
 logs:
   path:     "/tmp/"
+  level:    debug
 
-  default:  "rftpd-environment.log"
+  thin:     "rftpd-thin.log"
+  default:  "rftpd-core.log"
 
-  thin:     "rftpd-environment-thin.log"
-  newrelic: "rftpd-environment-newrelic.log"
-  queue:    "rftpd-environment-core.log"
-  api:      "rftpd-environment-core.log"
-  workers:  "rftpd-environment-core.log"
+  queue:    "rftpd-core.log"
+  api:      "rftpd-core.log"
+  workers:  "rftpd-core.log"
+  transfer: "rftpd-workers.log"
+  conchita: "rftpd-workers.log"
+  reporter: "rftpd-workers.log"
+  notify:   "rftpd-workers.log"
 
-  transfer: "rftpd-environment-workers.log"
-  conchita: "rftpd-environment-workers.log"
-  reporter: "rftpd-environment-workers.log"
-  notify:   "rftpd-environment-workers.log"
+  newrelic: "rftpd-newrelic.log"
+  rollbar:  "rftpd-rollbar.log"

data/lib/rest-ftp-daemon.rb

@@ -8,6 +8,7 @@ require "timeout"
 require "syslog"
 require "thread"
 require "newrelic_rpm"
+require "rollbar"
 
 # Shared libs / monkey-patching
 require_relative "shared/patch_array"
@@ -28,10 +29,10 @@ require_relative "rest-ftp-daemon/notification"
 require_relative "rest-ftp-daemon/location"
 
 # Remotes
-require_relative "rest-ftp-daemon/remote"
-require_relative "rest-ftp-daemon/remote_ftp"
-require_relative "rest-ftp-daemon/remote_sftp"
-require_relative "rest-ftp-daemon/remote_s3"
+require_relative "rest-ftp-daemon/remote/base"
+require_relative "rest-ftp-daemon/remote/ftp"
+require_relative "rest-ftp-daemon/remote/sftp"
+require_relative "rest-ftp-daemon/remote/s3"
 
 # Jobs
 require_relative "rest-ftp-daemon/job"
@@ -40,19 +41,18 @@ require_relative "rest-ftp-daemon/jobs/dummy"
 require_relative "rest-ftp-daemon/jobs/transfer"
 require_relative "rest-ftp-daemon/jobs/video"
 
-require_relative "rest-ftp-daemon/worker_pool"
-
 # Workers
 # require_from :workers
+require_relative "rest-ftp-daemon/worker_pool"
 require_relative "rest-ftp-daemon/workers/worker"
 require_relative "rest-ftp-daemon/workers/conchita"
 require_relative "rest-ftp-daemon/workers/reporter"
 require_relative "rest-ftp-daemon/workers/transfer"
 
-# API
-require_relative "rest-ftp-daemon/api/entities/location"
-require_relative "rest-ftp-daemon/api/entities/options"
-require_relative "rest-ftp-daemon/api/entities/job"
+# Entities and API
+require_relative "rest-ftp-daemon/entities/location"
+require_relative "rest-ftp-daemon/entities/options"
+require_relative "rest-ftp-daemon/entities/job"
 require_relative "rest-ftp-daemon/api/jobs"
 require_relative "rest-ftp-daemon/api/dashboard"
 require_relative "rest-ftp-daemon/api/status"
@@ -60,6 +60,10 @@ require_relative "rest-ftp-daemon/api/config"
 require_relative "rest-ftp-daemon/api/debug"
 require_relative "rest-ftp-daemon/api/root"
 
+# Init
+require_relative "rest-ftp-daemon/initialize"
+
+
 # def require_from subdir
 #   path = sprintf(
 #     '%s/rest-ftp-daemon/%s/*.rb',

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -59,12 +59,30 @@ module RestFtpDaemon
       end
 
       desc "Create a new job"
-      params do
-        requires :source, type: String, desc: "Source file pattern"
-        requires :target, type: String, desc: "Target remote path"
+     # desc 'Creates a new app' do
+     #    detail 'It is used to register a new app on the server and get the app_id'
+     #    params Entities::AppsParamsEntity.documentation
+     #    success Entities::AppsEntity
+     #    failure [[400, 'Bad Request', Entities::ErrorEntity]]
+     #    named 'create app'
+     #  end
 
-        optional :label, type: String, desc: "Descriptive label (info only)"
-        optional :notify, type: String, desc: "URL to get POST'ed notifications back"
+      params do
+        requires :source,
+          type: String,
+          desc: "Source file pattern",
+          allow_blank: false
+        requires :target,
+          type: String,
+          desc: "Target remote path",
+          allow_blank: false
+        optional :label,
+          type: String,
+          desc: "Descriptive label (info only)"
+        optional :notify,
+          type: String,
+          desc: "URL to get POST'ed notifications back",
+          allow_blank: false
         optional :type,
           type: String,
           desc: "Type of job",
@@ -81,19 +99,19 @@ module RestFtpDaemon
           default: 0
 
         optional :video_options, type: Hash, desc: "Options passed to FFMPEG encoder", default: {} do
-           optional :video_codec,             type: String
-           optional :video_bitrate,           type: String
-           optional :video_bitrate_tolerance, type: String
-           optional :frame_rate,              type: Integer
-           optional :resolution,              type: String
-           optional :aspect,                  type: String
-           optional :keyframe_interval,       type: String
-           optional :x264_vprofile,           type: String
-           optional :x264_preset,             type: String
-           optional :audio_codec,             type: String
-           optional :audio_bitrate,           type: String
-           optional :audio_sample_rate,       type: Integer
-           optional :audio_channels,          type: String
+           optional :video_codec,             type: String,  desc: "video codec (ex: copy, libx264)"
+           optional :video_bitrate,           type: String,  desc: "nominal video bitrate"
+           optional :video_bitrate_tolerance, type: String,  desc: "maximum video bitrate"
+           optional :frame_rate,              type: Integer, desc: "output frames per second"
+           optional :resolution,              type: String,  desc: "output video resolution"
+           optional :aspect,                  type: String,  desc: "output aspect ratio"
+           optional :keyframe_interval,       type: String,  desc: "group of pictures (GOP) size"
+           optional :x264_vprofile,           type: String,  desc: "h264 profile"
+           optional :x264_preset,             type: String,  desc: "h264 preset (fast, low..)"
+           optional :audio_codec,             type: String,  desc: "audio codec (ex: copy, libfaac, ibfdk_aac)"
+           optional :audio_bitrate,           type: String,  desc: "nominal audio bitrate"
+           optional :audio_sample_rate,       type: Integer, desc: "audio sampling rate"
+           optional :audio_channels,          type: String,  desc: "number of audio channels"
         end
 
         optional :video_custom,
@@ -113,6 +131,12 @@ module RestFtpDaemon
           type: Boolean,
           desc: "Upload to a temp file before renaming it to the target filename",
           default: Conf.at(:transfer, :tempfile)
+        # given :shelf_id do
+        #   requires :bin_id, type: Integer
+        # end
+        # given category: ->(val) { val == 'foo' } do
+        #   requires :description
+        # end
       end
 
       post "/" do

data/lib/rest-ftp-daemon/api/root.rb

@@ -50,6 +50,11 @@ module RestFtpDaemon
         put "-----"
       end
 
+      ## GLOBAL EXCEPTION HANDLING
+      rescue_from :all do |exception|
+        Rollbar.error exception
+        error_response(message: "Internal server error: #{exception}", status: 500)
+      end
 
       ### MOUNTPOINTS
       mount RestFtpDaemon::API::Status      => MOUNT_STATUS
@@ -71,7 +76,6 @@ module RestFtpDaemon
           }
 
 
-
       ### INITIALIZATION
       def initialize
         super

data/lib/rest-ftp-daemon/constants.rb

@@ -20,7 +20,6 @@ INFO_PROGRESS             = :work_progress
 INFO_BITRATE              = :transfer_bitrate
 
 # Constants: logger
-LOG_ROTATION             = "daily"
 LOG_FORMAT_PROGNAME      = "%d\t%s"
 LOG_HEADER_TIME          = "%Y-%m-%d %H:%M:%S"
 LOG_HEADER_FORMAT        = "%s \t%d\t%-8s %-10s "

data/lib/rest-ftp-daemon/exceptions.rb

@@ -15,6 +15,7 @@ module RestFtpDaemon
   class AttributeMissing          < RestFtpDaemonException; end
   class AssertionFailed           < RestFtpDaemonException; end
   class UnresolvedTokens          < RestFtpDaemonException; end
+  class LocationMalformed         < RestFtpDaemonException; end
   class LocationParseError        < RestFtpDaemonException; end
   class UnsupportedScheme         < RestFtpDaemonException; end
   class MissingFfmpegLibraries    < RestFtpDaemonException; end

data/lib/rest-ftp-daemon/helpers/views.rb

@@ -68,10 +68,15 @@ module RestFtpDaemon
     end
 
     def job_type job
+      # sprintf(
+      #     '<span class="glyphicon glyphicon-%s" alt="%s"></span>&nbsp;%s',
+      #     job_style(job),
+      #     job.type,
+      #     job.type
+      #     )
       sprintf(
-          '<span class="glyphicon glyphicon-%s" alt="%s"></span>&nbsp;%s',
+          '<span class="glyphicon glyphicon-%s" alt="%s"></span>',
           job_style(job),
-          job.type,
           job.type
           )
     end

data/lib/rest-ftp-daemon/initialize.rb

@@ -0,0 +1,3 @@
+# Init Rollbar and Newrelic
+Conf.prepare_newrelic
+Conf.prepare_rollbar

data/lib/rest-ftp-daemon/job.rb

@@ -303,6 +303,9 @@ module RestFtpDaemon
         log_error message
       end
 
+      # Log to Rollbar
+      Rollbar.error(exception)
+
       # Close ftp connexion if open
       @remote.close unless @remote.nil? || !@remote.connected?
 

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -22,16 +22,16 @@ module RestFtpDaemon
       case target_uri
       when URI::FTP
         log_info "JobTransfer.before target_method FTP"
-        @remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
+        @remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
       when URI::FTPES, URI::FTPS
         log_info "JobTransfer.before target_method FTPES/FTPS"
-        @remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
+        @remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
       when URI::SFTP
         log_info "JobTransfer.before target_method SFTP"
-        @remote = RemoteSFTP.new @target_loc, log_prefix, @config[:debug_sftp]
+        @remote = Remote::RemoteSFTP.new @target_loc, log_prefix, @config[:debug_sftp]
       when URI::S3
         log_info "JobTransfer.before target_method S3"
-        @remote = RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
+        @remote = Remote::RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
       else
         log_info "JobTransfer.before unknown scheme [#{@target_loc.scheme}]"
         raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme

data/lib/rest-ftp-daemon/location.rb

@@ -27,6 +27,10 @@ module RestFtpDaemon
       to: :uri
 
     def initialize original
+      unless original.is_a? String
+        raise RestFtpDaemon::LocationMalformed, original.inspect
+      end
+
       # Strip spaces before/after, copying original "path" at the same time
       @original = original
       location_uri = original.strip
@@ -102,6 +106,7 @@ module RestFtpDaemon
 
       # Replace endpoints defined in config
       vectors.each do |from, to|
+        next unless to.is_a? String
         next if to.to_s.empty?
         path.gsub! tokenize(from), to
       end