rest-ftp-daemon 0.90.1 → 0.94.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1f8cd7be9c2f0d1932b820c3cf38b7dc79de537
-  data.tar.gz: e001c9fb6c9299f3d1f49b15368fac6628fb0fd9
+  metadata.gz: eb821dc634f6867b5475e98868b9e6562117ebd2
+  data.tar.gz: 953acf9755629a9ebae2afaac8f1f54438358afb
 SHA512:
-  metadata.gz: 0f37294ceee1337337cbc4013fd1f8fa08438d5b6ea11fc698cfa1c59442864f4b5cbe7444deedb265e549a5c9a0c32b2e7b4e845c61c1a659656206369f2045
-  data.tar.gz: 28c38f2a5e8618abf1f24773b91381f4623843b9629a190c967641c4980ebc78b9306d6ef26b4bee2da65b6ce616f40f043f5756892b06356b01c9589e15a4f5
+  metadata.gz: e1f92e85d401a25533c0239e8798ea71cc6d3862aec7e176c6ff3dd8486396cab93977abd7d94866820a165ac23f4e4e1a6db7f218cb58db6db4aa43ca79875e
+  data.tar.gz: 6d6d0b77b70bf07a84a39918be28502f45ae71ab8ea6641dbf7ecec9aaa667d212c07963294c2e09f24dd8e91770d7af9d10ddc46928d3750b9aed4baa57eef5

data/.ruby-version

@@ -1 +1 @@
-2.1.0
+2.1.5

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.88)
+    rest-ftp-daemon (0.92)
       double-bag-ftps
       facter
       grape
+      grape-entity
       haml
       json
       settingslogic
@@ -16,7 +17,7 @@ GEM
   remote: http://rubygems.org/
   specs:
     CFPropertyList (2.2.8)
-    activesupport (4.1.7)
+    activesupport (4.1.8)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -48,13 +49,16 @@ GEM
       rack-accept
       rack-mount
       virtus (>= 1.0.0)
+    grape-entity (0.4.4)
+      activesupport
+      multi_json (>= 1.3.2)
     haml (4.0.5)
       tilt
     hashie (3.3.1)
     i18n (0.6.11)
     ice_nine (0.11.0)
     json (1.8.1)
-    minitest (5.4.2)
+    minitest (5.4.3)
     multi_json (1.10.1)
     multi_xml (0.5.5)
     rack (1.5.2)
@@ -62,7 +66,7 @@ GEM
       rack (>= 0.4)
     rack-mount (0.8.3)
       rack (>= 1.0.0)
-    rake (10.3.2)
+    rake (10.4.0)
     settingslogic (2.0.9)
     sys-cpu (0.7.1)
       ffi (>= 1.0.0)

data/README.md

@@ -184,6 +184,22 @@ GET http://localhost:3100/jobs/1
 ```
 
 
+
+Doc TODO
+------------------------------------------------------------------------------------
+
+Document /status
+
+Document /routes
+
+Document multiple-files upload
+
+Document mkdir and overwrite options
+
+Document counters
+
+
+
 About
 ------------------------------------------------------------------------------------
 

data/bin/rest-ftp-daemon

@@ -63,7 +63,7 @@ puts Settings.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion =>
 puts
 
 # Validate network configuration
-if ["start", "restart"].include? command
+if ["start"].include? command
   if Settings['port'].nil?
     puts "ABORTING: Network port is missing"
     exit 1

data/lib/rest-ftp-daemon.rb

@@ -1,6 +1,13 @@
 # Global libs
 require 'rubygems'
 require 'json'
+require 'grape'
+require 'grape-entity'
+require 'haml'
+require "facter"
+require "sys/cpu"
+
+
 # require 'celluloid/autostart'
 
 # My libs
@@ -15,7 +22,10 @@ require 'rest-ftp-daemon/worker_pool'
 require 'rest-ftp-daemon/logger'
 require 'rest-ftp-daemon/job'
 require 'rest-ftp-daemon/notification'
-require 'rest-ftp-daemon/api/defaults'
-require 'rest-ftp-daemon/api/jobs'
 require 'rest-ftp-daemon/api/root'
-
+require 'rest-ftp-daemon/api/jobs'
+require 'rest-ftp-daemon/api/debug'
+require 'rest-ftp-daemon/api/routes'
+require 'rest-ftp-daemon/api/dashboard'
+require 'rest-ftp-daemon/api/status'
+require 'rest-ftp-daemon/api/job_presenter'

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -0,0 +1,60 @@
+module RestFtpDaemon
+  module API
+    class Root < Grape::API
+
+####### DASHBOARD - GET /
+
+      # Server global status
+      get '/' do
+        info "GET /"
+
+        # Initialize UsageWatch
+        Facter.loadfacts
+        @info_load = Sys::CPU.load_avg.first.to_f
+        @info_procs = (Facter.value :processorcount).to_i
+        @info_ipaddr = Facter.value(:ipaddress)
+        @info_memfree = Facter.value(:memoryfree)
+
+        # Compute normalized load
+        if @info_procs.zero?
+          @info_norm = "N/A"
+        else
+          @info_norm = (100 * @info_load / @info_procs).round(1)
+        end
+
+        # Jobs to display
+        popped_jobs = $queue.ordered_popped.reverse
+        @jobs_queued = $queue.ordered_queue.reverse
+
+        if params["only"].nil? || params["only"].blank?
+          @only = nil
+        else
+          @only = params["only"].to_sym
+        end
+
+        if @only.nil?
+          @jobs_popped = popped_jobs
+        else
+          @jobs_popped = $queue.popped_reverse_sorted_by_status @only
+        end
+
+        # Count jobs for each status
+        @counts = $queue.counts_by_status
+
+        # Get workers status
+        @gworker_statuses = $pool.get_worker_statuses
+
+        # Compile haml template
+        output = render :dashboard
+
+        # Send response
+        env['api.format'] = :html
+        format "html"
+        status 200
+        content_type "text/html"
+        body output
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/debug.rb

@@ -0,0 +1,25 @@
+module RestFtpDaemon
+  module API
+    class Root < Grape::API
+
+####### GET /debug
+
+      get '/debug' do
+        info "GET /debug"
+        begin
+          raise RestFtpDaemon::DummyException
+        rescue RestFtpDaemon::RestFtpDaemonException => exception
+          status 501
+          api_error exception
+        rescue Exception => exception
+          status 501
+          api_error exception
+        else
+          status 200
+          {}
+        end
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -0,0 +1,33 @@
+module RestFtpDaemon
+  module API
+    module Entities
+
+      class JobPresenter < Grape::Entity
+        # Job ID
+        expose :id
+
+        # Job specific attributes
+        Job::FIELDS.each do |field|
+          expose field
+          #expose field, unless: lambda { |object, options| object.instance_variable_get("@#{field}").nil? }
+        end
+
+        # Technical fields
+        expose :wid, unless: lambda { |object, options| object.wid.nil? }
+
+        expose :error
+        expose :status
+
+        expose :started_at
+        expose :updated_at
+        expose :age
+
+        # Params
+        # expose :wid, unless: lambda { |object, options| object.wid.nil? }
+        expose :params, unless: :hide_params
+
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -1,84 +1,19 @@
 module RestFtpDaemon
   module API
+    class Root < Grape::API
 
-    class Jobs < Grape::API
 
+####### GET /jobs/:id
 
-####### CLASS CONFIG
-
-      # params do
-      #   optional :overwrite, type: Integer, default: false
-      # end
-
-
-####### INITIALIZATION
-
-      def initialize
-        #$last_worker_id = 0
-
-        # Check that Queue and Pool are available
-        raise RestFtpDaemon::MissingQueue unless defined? $queue
-        raise RestFtpDaemon::MissingPool unless defined? $pool
-
-        super
-      end
-
-
-####### HELPERS
-
-      helpers do
-
-        def threads_with_id job_id
-          $threads.list.select do |thread|
-            next unless thread[:job].is_a? Job
-            thread[:job].id == job_id
-          end
-        end
-
-        def job_describe job_id
-          raise RestFtpDaemon::JobNotFound if ($queue.all_size==0)
-
-          # Find job with exactly this id
-          found = $queue.find_by_id(job_id)
-
-          # Find job with this id while searching with the current prefix
-          found = $queue.find_by_id(job_id, true) if found.nil?
-
-          # Check that we did find it
-          raise RestFtpDaemon::JobNotFound if found.nil?
-          raise RestFtpDaemon::JobNotFound unless found.is_a? Job
-
-          # Return job description
-          found.describe
-        end
-
-        # def job_delete job_id
-        # end
-
-        def job_list
-          $queue.all.map do |item|
-            next unless item.is_a? Job
-            item.describe
-          end
-        end
-
-      end
-
-
-####### API DEFINITION
-
-      desc "Get information about a specific job"
       params do
-        requires :id, type: String, desc: "job id", regexp:  /[^\/]+/
-        # optional :audio do
-        #   requires :format, type: Symbol, values: [:mp3, :wav, :aac, :ogg], default: :mp3
-        # end
+        requires :id, type: String, desc: 'ID of the Job to read', regexp: /[^\/]+/
       end
-      get '*id' do
+      get '/jobs/*id' do
         info "GET /jobs/#{params[:id]}"
-        #return params
         begin
-          response = job_describe params[:id]
+          job = job_find params[:id]
+          raise RestFtpDaemon::JobNotFound if job.nil?
+
         rescue RestFtpDaemon::JobNotFound => exception
           status 404
           api_error exception
@@ -90,38 +25,20 @@ module RestFtpDaemon
           api_error exception
         else
           status 200
-          response
+          present job, :with => RestFtpDaemon::API::Entities::JobPresenter, type: "complete"
         end
-      end
 
-      # Delete jobs
-      desc "Kill and remove a specific job"
-      delete ':id' do
-       info "DELETE /jobs/#{params[:name]}"
-       status 501
-        # begin
-        #   response = job_delete params[:id].to_i
-        # rescue RestFtpDaemon::JobNotFound => exception
-        #   status 404
-        #   api_error exception
-        # rescue RestFtpDaemonException => exception
-        #   status 500
-        #   api_error exception
-        # rescue Exception => exception
-        #   status 501
-        #   api_error exception
-        # else
-        #   status 200
-        #   response
-        # end
       end
 
-      # List jobs
-      desc "Get a list of jobs"
-      get do
+
+####### GET /jobs/
+
+      desc "List all Jobs"
+
+      get '/jobs/' do
         info "GET /jobs"
         begin
-          response = job_list
+          jobs = $queue.all
         rescue RestFtpDaemonException => exception
           status 501
           api_error exception
@@ -130,33 +47,42 @@ module RestFtpDaemon
           api_error exception
         else
           status 200
-          response
+          present jobs, :with => RestFtpDaemon::API::Entities::JobPresenter
         end
       end
 
 
-      # Spawn a new thread for this new job
+
+####### POST /jobs/
+
       desc "Create a new job"
-      post do
-        info "POST /jobs: #{request.body.read}"
-        begin
-          # Extract params
-          request.body.rewind
-          params = JSON.parse(request.body.read, symbolize_names: true)
 
+      params do
+        requires :source, type: String, desc: "Source file pattern"
+        requires :target, type: String, desc: "Target remote path"
+        optional :notify, type: String, desc: "URL to get POST'ed notifications back"
+        optional :priority, type: Integer, desc: "Priority level of the job (lower is stronger)"
+        optional :overwrite, type: Boolean, desc: "wether to overwrites files at target server",
+          default: false
+        optional :mkdir, type: Boolean, desc: "wether to create missing directories on target server",
+          default: false
+      end
 
-params[:priority] = rand(10)
+      post '/jobs/' do
+        info "POST /jobs #{params.inspect}"
+        # request.body.rewind
+        begin
 
           # Create a new job
-          # job_id = $last_worker_id += 1
           job_id = $queue.generate_id
-          #job = Job.new(params)
           job = Job.new(job_id, params)
 
           # And push it to the queue
-          #$queue.push0 job
           $queue.push job
 
+          # Increment a counter
+          $queue.counter_inc :jobs_received
+
         rescue JSON::ParserError => exception
           status 406
           api_error exception
@@ -168,15 +94,10 @@ params[:priority] = rand(10)
           api_error exception
         else
           status 201
-          job.describe
+          present job, :with => RestFtpDaemon::API::Entities::JobPresenter
         end
       end
 
-    protected
-
-    # def progname
-    #   "API::Jobs"
-    # end
 
     end
   end