rest-ftp-daemon 0.72b → 0.85.2

data/lib/rest-ftp-daemon.rb

@@ -1,13 +1,14 @@
 # Global libs
 require 'rubygems'
 require 'json'
-require 'securerandom'
 # require 'celluloid/autostart'
 
 # My libs
+require 'rest-ftp-daemon/constants'
 require 'rest-ftp-daemon/config'
 require 'rest-ftp-daemon/exceptions'
 require 'rest-ftp-daemon/common'
+require 'rest-ftp-daemon/helpers'
 require 'rest-ftp-daemon/uri'
 require 'rest-ftp-daemon/job_queue'
 require 'rest-ftp-daemon/worker_pool'

data/lib/rest-ftp-daemon/api/defaults.rb

@@ -1,7 +1,5 @@
 # encoding: UTF-8
 require 'grape'
-SIZE_UNITS = ["B", "KB", "MB", "GB", "TB", "PB" ]
-
 
 module RestFtpDaemon
   module API
@@ -45,13 +43,6 @@ module RestFtpDaemon
         # end
 
         helpers do
-          def format_nice_bytes( number )
-            return "Ø" if number.nil? || number.zero?
-            index = ( Math.log( number ) / Math.log( 2 ) ).to_i / 10
-            converted = number.to_i / ( 1024 ** index )
-            "#{converted} #{SIZE_UNITS[index]}"
-          end
-
           def api_error exception
             {
             :error => exception.class,
@@ -62,7 +53,7 @@ module RestFtpDaemon
           end
 
           def render name, values={}
-            template = File.read("#{Settings.app_lib}/views/#{name.to_s}.haml")
+            template = File.read("#{APP_LIBS}/views/#{name.to_s}.haml")
             haml_engine = Haml::Engine.new(template)
             haml_engine.render(binding, values)
           end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -6,22 +6,19 @@ module RestFtpDaemon
 
 ####### CLASS CONFIG
 
-      #include RestFtpDaemon::API::Defaults
-      #logger ActiveSupport::Logger.new Settings.logs.api, 'daily' unless Settings.logs.api.nil?
-
-      params do
-        optional :overwrite, type: Integer, default: false
-      end
+      # params do
+      #   optional :overwrite, type: Integer, default: false
+      # end
 
 
 ####### INITIALIZATION
 
       def initialize
-        $last_worker_id = 0
+        #$last_worker_id = 0
 
         # Check that Queue and Pool are available
         raise RestFtpDaemon::MissingQueue unless defined? $queue
-        raise RestFtpDaemon::MissingQueue unless defined? $pool
+        raise RestFtpDaemon::MissingPool unless defined? $pool
 
         super
       end
@@ -31,6 +28,13 @@ module RestFtpDaemon
 
       helpers do
 
+        def next_job_id
+          @@last_worker_id ||= 0
+          @@last_worker_id += 1
+          #Helpers.identifier(IDENT_JOB_BYTES)
+          #SecureRandom.hex(IDENT_JOB_BYTES)
+        end
+
         def threads_with_id job_id
           $threads.list.select do |thread|
             next unless thread[:job].is_a? Job
@@ -39,7 +43,7 @@ module RestFtpDaemon
         end
 
         def job_describe job_id
-          raise RestFtpDaemon::JobNotFound if ($queue.queued_size==0 && $queue.popped_size==0)
+          raise RestFtpDaemon::JobNotFound if ($queue.all_size==0)
 
           # Find job with this id
           found = $queue.all.select { |job| job.id == job_id }.first
@@ -136,18 +140,16 @@ module RestFtpDaemon
         begin
           # Extract params
           request.body.rewind
-          params = JSON.parse request.body.read
+          params = JSON.parse(request.body.read, symbolize_names: true)
 
           # Create a new job
-          job_id = $last_worker_id += 1
+          # job_id = $last_worker_id += 1
+          job_id = next_job_id
           job = Job.new(job_id, params)
 
-          # And psuh it to the queue
+          # And push it to the queue
           $queue.push job
 
-          # Later: start it asynchronously
-          #job.future.process
-
         rescue JSON::ParserError => exception
           status 406
           api_error exception
@@ -165,9 +167,9 @@ module RestFtpDaemon
 
     protected
 
-    def progname
-      "API::Jobs"
-    end
+    # def progname
+    #   "API::Jobs"
+    # end
 
     end
   end

data/lib/rest-ftp-daemon/api/root.rb

@@ -10,11 +10,11 @@ module RestFtpDaemon
 
 
 ####### CLASS CONFIG
-      include RestFtpDaemon::API::Defaults
-      logger ActiveSupport::Logger.new(Settings.logs.api, 'daily') unless Settings.logs.api.nil?
-      #add_swagger_documentation
 
+      include RestFtpDaemon::API::Defaults
+      logger RestFtpDaemon::Logger.new(:api, "API")
       mount RestFtpDaemon::API::Jobs => '/jobs'
+      #add_swagger_documentation
       # mount RestFtpDaemon::API::Workers => '/workers'
 
 
@@ -22,7 +22,7 @@ module RestFtpDaemon
 
       helpers do
         def info message, level = 0
-          Root.logger.add(Logger::INFO, "#{'  '*level} #{message}", "API::Root")
+          Root.logger.info(message, level)
         end
 
         def job_list_by_status
@@ -32,7 +32,6 @@ module RestFtpDaemon
             statuses[item.get_status] ||= 0
             statuses[item.get_status] +=1
           end
-
           statuses
         end
 
@@ -42,12 +41,9 @@ module RestFtpDaemon
 ####### INITIALIZATION
 
       def initialize
-        $last_worker_id = 0
-
         # Check that Queue and Pool are available
         raise RestFtpDaemon::MissingQueue unless defined? $queue
         raise RestFtpDaemon::MissingQueue unless defined? $pool
-
         super
       end
 
@@ -56,10 +52,7 @@ module RestFtpDaemon
 
       # Server global status
       get '/' do
-        # Prepare data
-        @jobs_all = $queue.all
-        #@jobs_all_size = $queue.all_size
-        #@jobs_all = $queue.all_size
+        info "GET /"
 
         # Initialize UsageWatch
         Facter.loadfacts
@@ -68,24 +61,42 @@ module RestFtpDaemon
         @info_ipaddr = Facter.value(:ipaddress)
         @info_memfree = Facter.value(:memoryfree)
 
-
         # Compute normalized load
-        # puts "info_procs: #{info_procs}"
         if @info_procs.zero?
           @info_norm = "N/A"
         else
           @info_norm = (100 * @info_load / @info_procs).round(1)
         end
 
-        # Compute total transferred
-        @total_transferred = 0
-        @jobs_all.each do |job|
-          sent = job.get(:file_sent)
-          @total_transferred += sent unless sent.nil?
+        # Jobs to display
+        all_jobs_in_queue = $queue.all
+
+        if params["only"].nil? || params["only"].blank?
+          @only = nil
+        else
+          @only = params["only"].to_sym
         end
 
+        case @only
+        when nil
+          @jobs = all_jobs_in_queue
+        when :queue
+          @jobs = $queue.queued
+        else
+          @jobs = $queue.by_status (@only)
+        end
+
+        # Count jobs for each status
+        @counts = {}
+        grouped = all_jobs_in_queue.group_by { |job| job.get(:status) }
+        grouped.each do |status, jobs|
+          @counts[status] = jobs.size
+        end
+
+        # Get workers status
+        @gworker_statuses = $pool.get_worker_statuses
+
         # Compile haml template
-        @name = "Test"
         output = render :dashboard
 
         # Send response
@@ -99,14 +110,14 @@ module RestFtpDaemon
 
       # Server global status
       get '/status' do
-        info "GET /"
+        info "GET /status"
         status 200
         return  {
           hostname: `hostname`.chomp,
-          version: Settings.app_ver,
+          version: APP_VER,
           config: Settings.to_hash,
-          started: Settings.app_started,
-          uptime: (Time.now - Settings.app_started).round(1),
+          started: APP_STARTED,
+          uptime: (Time.now - APP_STARTED).round(1),
           status: job_list_by_status,
           queue_size: $queue.all_size,
           jobs_queued: $queue.queued.collect(&:id),
@@ -117,7 +128,8 @@ module RestFtpDaemon
 
       # Server test
       get '/debug' do
-        info "GET /debug/"
+        info "GET /debug"
+
         begin
           raise RestFtpDaemon::DummyException
         rescue RestFtpDaemon::RestFtpDaemonException => exception

data/lib/rest-ftp-daemon/common.rb

@@ -4,45 +4,10 @@ module RestFtpDaemon
 
   protected
 
-    def initialize
-      # Logger
-      @logger = ActiveSupport::Logger.new Settings.logs.workers, 'daily' unless Settings.logs.workers.nil?
-    end
-
-    def id
-    end
-
-    def progname
-    end
+    # FIXME: should be moved to class itself to get rid of this parent class
 
     def info message, level = 0
-      # progname = "Job [#{id}]" unless id.nil?
-      # progname = "Worker [#{id}]" unless worker_id.nil?
-      @logger.add(Logger::INFO, "#{'  '*(level+1)} #{message}", progname) unless @logger.nil?
-    end
-
-    def notify signal, error = 0, status = {}
-      # Skip is not callback URL defined
-      url = get :notify
-      if url.nil?
-        info "Skipping notification (no valid url provided) sig[#{signal}] e[#{error}] s#{status.inspect}"
-        return
-      end
-
-      # Build notification
-      n = RestFtpDaemon::Notification.new
-      n.job_id = id
-      n.url = url
-      n.signal = signal
-      n.error = error.inspect
-      n.status = status
-
-      # Now, send the notification
-      info "Queuing notification key[#{n.key}] sig[#{signal}] url[#{url}]"
-      Thread.new(n) do |thread|
-        n.notify
-      end
-
+      @logger.info(message, level) unless @logger.nil?
     end
 
   end

data/lib/rest-ftp-daemon/config.rb

@@ -1,26 +1,21 @@
-require 'settingslogic'
-
-# Terrific assertions
-#raise "config.rb: APP_ROOT is not defined" unless defined? APP_ROOT
-APP_NAME = "rest-ftp-daemon"
-APP_CONF = "/etc/#{APP_NAME}.yml"
-APP_DEV = ARGV.include?("development") ? true : false
+# Try to load Settingslogic
+begin
+  require "settingslogic"
+rescue LoadError
+  raise "config.rb warning: Settingslogic is needed to provide configuration values to the Gemspec file"
+end
 
+# Configuration class
 class Settings < Settingslogic
   # Read configuration
-  source (File.exists? APP_CONF) ? APP_CONF : Hash.new
-  namespace (APP_DEV ? "development" : "production")
+  namespace (defined?(APP_ENV) ? APP_ENV : "production")
+  source ((File.exists? APP_CONF) ? APP_CONF : Hash.new)
   suppress_errors true
 
-  # Some constants
-  self[:dev] = APP_DEV
-  self[:app_name] = APP_NAME
-  self[:app_lib] = File.expand_path File.dirname(__FILE__)
-  self[:app_ver] = "0.72b"
-  self[:app_started] = Time.now
-  self[:default_trim_progname] = "18"
+  # Compute my PID filename
+  def pidfile
+    self["pidfile"] || "/tmp/#{APP_NAME}.port#{self['port'].to_s}.pid"
+  end
 
-  # Some defaults
-  self[:default_chunk_size] = "1000000"
-  self[:default_notify_size] = "10000000"
 end
+

data/lib/rest-ftp-daemon/constants.rb

@@ -0,0 +1,21 @@
+# Terrific constants
+APP_NAME = "rest-ftp-daemon"
+APP_CONF = "/etc/#{APP_NAME}.yml"
+APP_VER = "0.85.2"
+
+
+# Some global constants
+IDENT_NOTIF_LEN = 4
+IDENT_RANDOM_LEN = 8
+
+
+# Some defaults
+DEFAULT_CONNECT_TIMEOUT_SEC = 30
+DEFAULT_UPDATE_EVERY_KB = 2048
+DEFAULT_WORKERS = 1
+DEFAULT_LOGS_PROGNAME_TRIM = 9
+
+
+# Initialize markers
+APP_STARTED = Time.now
+APP_LIBS = File.dirname(__FILE__)

data/lib/rest-ftp-daemon/exceptions.rb

@@ -1,33 +1,20 @@
 module RestFtpDaemon
 
-  class RestFtpDaemonException < StandardError; end
-
-  class DummyException < RestFtpDaemonException; end
-
-  class MissingQueue   < RestFtpDaemonException; end
-  class MissingPool   < RestFtpDaemonException; end
-
-
-  class RequestSourceMissing     < RestFtpDaemonException; end
-  class RequestSourceNotFound    < RestFtpDaemonException; end
-  class RequestTargetMissing     < RestFtpDaemonException; end
-  class RequestTargetScheme      < RestFtpDaemonException; end
-
-  class JobPrerequisitesNotMet   < RestFtpDaemonException; end
-
-  class JobNotFound              < RestFtpDaemonException; end
-  class JobSourceMissing         < RestFtpDaemonException; end
-  class JobSourceNotFound        < RestFtpDaemonException; end
-  class JobTargetMissing         < RestFtpDaemonException; end
-  class JobTargetUnsupported     < RestFtpDaemonException; end
-  class JobTargetUnparseable     < RestFtpDaemonException; end
-  #class JobTargetPermission      < RestFtpDaemonException; end
-  class JobTargetFileExists      < RestFtpDaemonException; end
-  class JobPrerequisitesNotMet      < RestFtpDaemonException; end
-
-
-  class NotificationMissingUrl   < RestFtpDaemonException; end
-  class NotificationMissingSignal   < RestFtpDaemonException; end
-
+  class RestFtpDaemonException    < StandardError; end
+
+  class DummyException            < RestFtpDaemonException; end
+
+  class MissingQueue              < RestFtpDaemonException; end
+  class MissingPool               < RestFtpDaemonException; end
+
+  class JobException              < RestFtpDaemonException; end
+  class JobNotFound               < RestFtpDaemonException; end
+  class JobAssertionFailed        < RestFtpDaemonException; end
+  class JobMissingAttribute       < RestFtpDaemonException; end
+  class JobSourceNotFound         < RestFtpDaemonException; end
+  class JobTargetUnsupported      < RestFtpDaemonException; end
+  class JobTargetUnparseable      < RestFtpDaemonException; end
+  class JobTargetFileExists       < RestFtpDaemonException; end
+  class JobTooManyOpenFiles       < RestFtpDaemonException; end
 
 end

data/lib/rest-ftp-daemon/helpers.rb

@@ -0,0 +1,55 @@
+require 'securerandom'
+
+
+module RestFtpDaemon
+  class Helpers
+
+    def self.format_bytes number, unit=""
+      return "Ø" if number.nil? || number.zero?
+
+      units = ["", "k", "M", "G", "T", "P" ]
+      index = ( Math.log( number ) / Math.log( 2 ) ).to_i / 10
+      converted = number.to_i / ( 1024 ** index )
+      "#{converted} #{units[index]}#{unit}"
+    end
+
+    def self.identifier len
+      rand(36**len).to_s(36)
+    end
+    #SecureRandom.hex(IDENT_JOB_BYTES)
+
+    def self.tokenize(item)
+      "[#{item}]"
+    end
+
+    def self.local_port_used? port
+      ip = '0.0.0.0'
+      timeout = 1
+      begin
+        Timeout::timeout(timeout) do
+          begin
+            TCPSocket.new(ip, port).close
+            true
+          rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+            false
+          rescue Errno::EADDRNOTAVAIL
+            "Settings.local_port_used: Errno::EADDRNOTAVAIL"
+          end
+        end
+      rescue Timeout::Error
+        false
+      end
+    end
+
+    # def snakecase
+    #   gsub(/::/, '/').
+    #   gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    #   gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    #   tr('-', '_').
+    #   gsub(/\s/, '_').
+    #   gsub(/__+/, '_').
+    #   downcase
+    # end
+
+  end
+end