rest-ftp-daemon 0.400.0 → 0.410.0.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ddc0fb0b76fb6590512af92bcff70b63b03d770
-  data.tar.gz: e43a27f7cc877d49bc965493aedd7a97cad1ff3a
+  metadata.gz: aef3a9c63e1ea02a0a3c79704750ab76ee2c8808
+  data.tar.gz: 9caa5c76fda9ae48b13a74d668967648a2c50e1c
 SHA512:
-  metadata.gz: 0b39316f2dc7892f5b77fb27a6e7f9b46df8dbb43be7382218ca25d49681114be8d4e5b99ef8e59e341d18da62302322b1431bb789a8dbc11b15bdbdf241f53e
-  data.tar.gz: b2a2e0a9c4f1a1737ce42e8e767c8ae3ffa19dfa30e12b63ebde9fd9492aa4f835436ed797c2fd93557392f5122a0f6a62255da738bd759faf57efa34ea9515a
+  metadata.gz: d24876af6c2818633132aef618fe808c1f09d4f624c01c436dfcd1217e455eaaa851fd07381cbe9f8f3b3f9aa54bd6ea54384f8ce8dcb556a29acb32f9e96349
+  data.tar.gz: b2de79768b659cec1428f37ef3ae42f23d7c0ff6c8c282c8bf148a8e178017f7ac2d87a538849f8d6f554d3c8a7f0b59651f0edf291f5db20cc111f4e86ccd71

data/Gemfile.lock

@@ -1,21 +1,25 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.400.0)
+    rest-ftp-daemon (0.410.0.pre.1)
       activesupport (~> 4.2)
       api-auth
-      bmc-daemon-lib (~> 0.3.3)
+      bmc-daemon-lib (~> 0.3.4)
       double-bag-ftps
       facter
       get_process_mem
       grape
       grape-entity
+      grape-swagger
+      grape-swagger-entity
+      grape-swagger-representable
       haml
       json (~> 1.8)
       net-sftp
       newrelic_rpm
       rest-client (~> 1.8)
       settingslogic
+      streamio-ffmpeg
       sys-cpu
       thin (~> 1.7)
 
@@ -38,20 +42,22 @@ GEM
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    bmc-daemon-lib (0.3.3)
+    bmc-daemon-lib (0.3.4)
       chamber (~> 2.9)
     builder (3.2.2)
-    chamber (2.9.0)
+    chamber (2.9.1)
       hashie (~> 3.3)
       thor (~> 0.19.1)
     coderay (1.1.1)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     daemons (1.2.4)
+    declarative (0.0.8)
+      uber (>= 0.0.15)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
-    domain_name (0.5.20160615)
+    domain_name (0.5.20160826)
       unf (>= 0.0.5, < 1.0.0)
     double-bag-ftps (0.1.3)
     enumerable-lazy (0.0.1)
@@ -74,6 +80,14 @@ GEM
     grape-entity (0.5.1)
       activesupport
       multi_json (>= 1.3.2)
+    grape-swagger (0.23.0)
+      grape (>= 0.12.0)
+    grape-swagger-entity (0.1.4)
+      grape-entity
+      grape-swagger (>= 0.20.4)
+    grape-swagger-representable (0.1.3)
+      grape-swagger (>= 0.20.2)
+      representable
     haml (4.0.7)
       tilt
     hashie (3.4.4)
@@ -100,7 +114,7 @@ GEM
       net-ssh (>= 2.6.5)
     net-ssh (3.2.0)
     netrc (0.11.0)
-    newrelic_rpm (3.16.1.320)
+    newrelic_rpm (3.16.2.321)
     parser (2.3.1.2)
       ast (~> 2.2)
     powerpack (0.1.1)
@@ -113,6 +127,9 @@ GEM
       rack (>= 0.4)
     rainbow (2.1.0)
     rake (11.2.2)
+    representable (3.0.0)
+      declarative (~> 0.0.5)
+      uber (~> 0.0.15)
     rest-client (1.8.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
@@ -139,6 +156,8 @@ GEM
     ruby-progressbar (1.8.1)
     settingslogic (2.0.9)
     slop (3.6.0)
+    streamio-ffmpeg (2.1.0)
+      multi_json (~> 1.8)
     sys-cpu (0.7.2)
       ffi
     thin (1.7.0)
@@ -150,6 +169,7 @@ GEM
     tilt (2.0.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
+    uber (0.0.15)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.2)

data/Rakefile

@@ -5,12 +5,14 @@ require "rubygems"
 require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
-# Run specs by default
-desc "Run all tests"
-
 require "rubocop/rake_task"
 RuboCop::RakeTask.new(:rubocop) do |task|
   task.fail_on_error = false
 end
 
+# Load my own tasks
+# Dir.glob('lib/tasks/*.rake').each { |r| import r }
+
+# Run specs by default
+desc "Run all tests"
 task default: [:spec, :rubocop]

data/bin/rest-ftp-daemon

@@ -89,7 +89,8 @@ end
 
 
 # Thin: basic options
-thin_logfile = Conf.at(:logs, :thin)
+thin_logfile = Conf.logfile_path(:thin)
+
 argv = []
 argv << ["-e",          Conf.app_env]
 argv << ["-p",          Conf[:port].to_s]

data/config.ru

@@ -12,7 +12,14 @@ unless Conf[:adminpwd].nil?
 end
 
 # Serve static assets
-use Rack::Static, urls: ["/css", "/js", "/images"], root: "#{Conf.app_libs}/static/"
+use Rack::Static, root: "#{Conf.app_libs}/static/", urls: [
+  "/css/",
+  "/js/",
+  "/fonts/",
+  "/images/",
+  "/swagger/",
+  MOUNT_SWAGGER_UI,
+  ]
 
 # Rack reloader and mini-profiler
 unless Conf.app_env == "production"
@@ -20,7 +27,6 @@ unless Conf.app_env == "production"
   # use Rack::MiniProfiler
 end
 
-
 # Initialize workers
 RestFtpDaemon::WorkerPool.instance.start_em_all
 

data/lib/rest-ftp-daemon.rb

@@ -26,21 +26,29 @@ require_relative "rest-ftp-daemon/paginate"
 require_relative "rest-ftp-daemon/uri"
 require_relative "rest-ftp-daemon/job_queue"
 require_relative "rest-ftp-daemon/counters"
-require_relative "rest-ftp-daemon/job"
 require_relative "rest-ftp-daemon/notification"
 
 require_relative "rest-ftp-daemon/path"
+require_relative "rest-ftp-daemon/location"
+
 require_relative "rest-ftp-daemon/remote"
 require_relative "rest-ftp-daemon/remote_ftp"
 require_relative "rest-ftp-daemon/remote_sftp"
 
+require_relative "rest-ftp-daemon/job"
+require_relative "rest-ftp-daemon/jobs/dummy"
+require_relative "rest-ftp-daemon/jobs/transfer"
+require_relative "rest-ftp-daemon/jobs/video"
+
 require_relative "rest-ftp-daemon/worker_pool"
 require_relative "rest-ftp-daemon/workers/worker"
 require_relative "rest-ftp-daemon/workers/conchita"
 require_relative "rest-ftp-daemon/workers/reporter"
 require_relative "rest-ftp-daemon/workers/transfer"
 
-require_relative "rest-ftp-daemon/api/job_presenter"
+require_relative "rest-ftp-daemon/api/entities/options"
+require_relative "rest-ftp-daemon/api/entities/job"
+
 require_relative "rest-ftp-daemon/api/jobs"
 require_relative "rest-ftp-daemon/api/dashboard"
 require_relative "rest-ftp-daemon/api/status"

data/lib/rest-ftp-daemon/api/debug.rb

@@ -23,7 +23,7 @@ module RestFtpDaemon
             me[:error] = job.error.encoding.to_s unless job.error.nil?
             me[:status] = job.status.encoding.to_s unless job.status.nil?
 
-            Job::FIELDS.each do |name|
+            RestFtpDaemon::Job::FIELDS.each do |name|
               value = job.send(name)
               me[name] = value.encoding.to_s if value.is_a? String
             end
@@ -37,13 +37,25 @@ module RestFtpDaemon
       end
 
       ### ENDPOINTS
-      desc "debug"
+      desc "debug"#, hidden: true
       get "/" do
-       # Build response
-       return  {
+
+        # Extract routes
+        routes = []
+        RestFtpDaemon::API::Root.routes.each do |route|
+          routes << {
+            url: "#{route.options[:method]} #{route.pattern.path}",
+            vars: route.instance_variables,
+            options: route.options
+            }
+
+        end
+
+        # Build response
+        return  {
           metrics: debug_metrics,
-          routes: RestFtpDaemon::API::Root.routes,
           encodings: debug_encodings,
+          routes: routes,
           }
       end
 

data/lib/rest-ftp-daemon/api/{job_presenter.rb → entities/job.rb}

@@ -3,12 +3,13 @@ require "grape-entity"
 module RestFtpDaemon
   module API
     module Entities
-      class JobPresenter < Grape::Entity
+      class Job < Grape::Entity
+
         # Job ID
         expose :id
 
         # Job specific attributes and flags
-        Job::FIELDS.each { |name| expose name }
+        RestFtpDaemon::Job::FIELDS.each { |name| expose name }
 
         # Technical fields
         expose :wid, unless: lambda { |object, _options| object.wid.nil? }
@@ -30,6 +31,28 @@ module RestFtpDaemon
         # Params
         expose :infos, unless: :hide_infos
 
+        # Options
+        # expose :options, using: API::Entities::Options
+        # expose :video_ac
+        # expose :video_custom
+
+        # with_options(format_with: :iso_timestamp) do
+        #     expose :created_at
+        #     expose :updated_at
+        # end
+
+        # expose :age do
+        # end
+
+        # expose :slots do |station,options|
+        #   station.slots.map{ |slot| SlotEntity.new(slot).serializable_hash }
+        # end
+
+      private
+
+        def priv_func
+        end
+
       end
     end
   end

data/lib/rest-ftp-daemon/api/entities/options.rb

@@ -0,0 +1,15 @@
+require "grape-entity"
+
+module RestFtpDaemon
+  module API
+    module Entities
+      class Options < Grape::Entity
+
+        # expose :opt1, documentation: { type: 'Boolean', desc: 'opt UN', required: false }
+        expose :opt2
+        # expose :opt6
+
+      end
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -5,7 +5,11 @@ module RestFtpDaemon
     class Jobs < Grape::API
 
       ### ENDPOINTS
-      desc "Read job with ID"
+      desc "Read job with ID", http_codes: [
+        { code: 200, message: "Here is the job you requested" },
+        { code: 404, message: "Job not found" }
+        ],
+        is_array: false
       params do
         requires :id, type: String, desc: "ID of the Job to read"
       end
@@ -16,6 +20,8 @@ module RestFtpDaemon
           job = RestFtpDaemon::JobQueue.instance.find_by_id(params[:id]) || RestFtpDaemon::JobQueue.instance.find_by_id(params[:id], true)
           raise RestFtpDaemon::JobNotFound if job.nil?
 
+          log_debug "found job: #{job.inspect}"
+
         rescue RestFtpDaemon::JobNotFound => exception
           log_error "JobNotFound: #{exception.message}"
           error!({ error: :api_job_not_found, message: exception.message }, 404)
@@ -26,12 +32,15 @@ module RestFtpDaemon
 
         else
           status 200
-          present job, with: RestFtpDaemon::API::Entities::JobPresenter, type: "complete"
+          present job, with: RestFtpDaemon::API::Entities::Job, type: "complete"
 
         end
       end
 
-      desc "List all Jobs"
+      desc "List all Jobs", http_codes: [
+        { code: 200, message: "Here are the jobs you requested" },
+        ],
+        is_array: true
       get "/" do
         begin
           # Get jobs to display
@@ -43,7 +52,7 @@ module RestFtpDaemon
 
         else
           status 200
-          present jobs, with: RestFtpDaemon::API::Entities::JobPresenter
+          present jobs, with: RestFtpDaemon::API::Entities::Job
 
         end
       end
@@ -52,10 +61,31 @@ module RestFtpDaemon
       params do
         requires :source, type: String, desc: "Source file pattern"
         requires :target, type: String, desc: "Target remote path"
+
         optional :label, type: String, desc: "Descriptive label for this job"
         optional :notify, type: String, desc: "URL to get POST'ed notifications back"
         optional :priority, type: Integer, desc: "Priority level of the job (lower is stronger)"
         optional :pool, type: String, desc: "Pool of worker to be used"
+        optional :type,
+          type: String,
+          desc: "Type of job",
+          default: JOB_TYPE_TRANSFER,
+          values: {value: JOB_TYPES, message: "should be one of: #{JOB_TYPES.join', '}"},
+          allow_blank: { value: false, message: 'cannot be empty' }
+
+        optional :video_vc,
+          type: String,
+          desc: "video: video codec",
+          default: ""
+        optional :video_ac,
+          type: String,
+          desc: "video: audio codec",
+          default: ""
+        optional :video_custom,
+          type: Hash,
+          desc: "video: custom options passed to FFMPEG encoder",
+          default: {}
+
         optional :overwrite,
           type: Boolean,
           desc: "Overwrites files at target server",
@@ -68,16 +98,15 @@ module RestFtpDaemon
           type: Boolean,
           desc: "Upload to a temp file before renaming it to the target filename",
           default: Conf.at(:transfer, :tempfile)
+        # optional :options, desc: "", type: API::Entities::Options
+
       end
+
       post "/" do
         # log_debug params.to_json
         begin
-          # Create a new job
-          job_id = RestFtpDaemon::JobQueue.instance.generate_id
-          job = Job.new(job_id, params)
-
-          # And push it to the queue
-          RestFtpDaemon::JobQueue.instance.push job
+          # Add up the new job on the queue
+          job = RestFtpDaemon::JobQueue.instance.create_job(params)
 
           # Increment a counter
           RestFtpDaemon::Counters.instance.increment :jobs, :received
@@ -86,13 +115,17 @@ module RestFtpDaemon
           log_error "JSON::ParserError: #{exception.message}"
           error!({error: :api_parse_error, message: exception.message}, 422)
 
+        rescue QueueCantCreateJob => exception
+          log_error "QueueCantCreateJob: #{exception.message}"
+          error!({error: :api_cant_create_job, message: exception.message}, 422)
+
         rescue RestFtpDaemonException => exception
-          log_error "RestFtpDaemonException: #{exception.message}"
-          error!({error: :api_exception, message: exception.message}, 500)
+          log_error "#{exception.class.to_s} #{exception.message}"
+          error!({error: exception_to_error(exception), message: exception.message}, 500)
 
         else
           status 201
-          present job, with: RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
+          present job, with: RestFtpDaemon::API::Entities::Job, hide_params: true
 
         end
       end

data/lib/rest-ftp-daemon/api/root-real.rb

@@ -0,0 +1,80 @@
+require "grape"
+require 'grape-swagger'
+
+module RestFtpDaemon
+  module API
+    class Root < Grape::API
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+      ### LOGGING & HELPERS
+      helpers RestFtpDaemon::CommonHelpers
+      helpers RestFtpDaemon::ApiHelpers
+      helpers BmcDaemonLib::LoggerHelper
+
+      helpers do
+        def log_prefix
+          ['API', nil, nil]
+        end
+
+        def logger
+          Root.logger
+        end
+      end
+
+      before do
+        log_request
+      end
+
+      ### CLASS CONFIG
+      logger BmcDaemonLib::LoggerPool.instance.get :api
+      do_not_route_head!
+      do_not_route_options!
+      # version 'v1'
+      format :json
+      content_type :json, 'application/json; charset=utf-8'
+
+
+desc 'API Root'
+
+      ### MOUNTPOINTS
+      # mount RestFtpDaemon::API::Status      => MOUNT_STATUS
+      # mount RestFtpDaemon::API::Jobs        => MOUNT_JOBS
+      # mount RestFtpDaemon::API::Dashbaord   => MOUNT_BOARD
+      # mount RestFtpDaemon::API::Config      => MOUNT_CONFIG
+      # mount RestFtpDaemon::API::Debug       => MOUNT_DEBUG
+
+
+      ### API Documentation
+      add_swagger_documentation hide_documentation_path: true,
+        api_version: BmcDaemonLib::Conf.app_ver,
+        doc_version: BmcDaemonLib::Conf.app_ver,
+        mount_path: MOUNT_SWAGGER_JSON,
+        info: {
+          title: BmcDaemonLib::Conf.app_name,
+          version: BmcDaemonLib::Conf.app_ver,
+          description: "API description for #{BmcDaemonLib::Conf.app_name} #{BmcDaemonLib::Conf.app_ver}",
+          }
+         # models: [
+         #   RestFtpDaemon::API::Entities::Job,
+         # ]
+
+      ### GLOBAL EXCEPTION HANDLING
+      # rescue_from :all do |e|
+      #   raise e
+      #   error_response(message: "Internal server error: #{e}", status: 500)
+      # end
+
+
+      ### INITIALIZATION
+      def initialize
+        super
+      end
+
+      ### ENDPOINTS
+      get "/" do
+        redirect dashboard_url()
+      end
+
+    end
+  end
+end