rest-ftp-daemon 0.95.2 → 0.100

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9df54e0c451475506949a750afee4f8c774b3ae7
-  data.tar.gz: 7d73e198e3b5e9d732db27937010cfa912057752
+  metadata.gz: 1cd7cf3fc402f4257b9c878b5227467f3f250c95
+  data.tar.gz: 441d0c886af8f6ba67347a82318efb40521862ce
 SHA512:
-  metadata.gz: 2c3f2f4e7331e3cfb09b0c61005d16429ea3a2a0ac54a223d61c6cb06b45b259ba6d90db6ddf75d851d1d4afc7f28c9f9a069a273795f3828c974783eb699492
-  data.tar.gz: 5cee703b40eb48e6bebc4e78b80d792fe4de5a024b70f6a9b1a72e4fbae56490abb84c3015fb420e6404a95035d650ff5a832aaae5883ffaf2cff19cd4dcda99
+  metadata.gz: e774def2fa7622368cca0d28915aaf6c45a121c438d6a4da34c408e733fc79c5611a0c72b1355c1200d5c49ec00d52b8a70198f243442d607e65f8b85ebbd7f7
+  data.tar.gz: 6696902d94a1130e840955a84c4cde5fac53b0df16bdebeb581c27ff7730efc12b22bf984a580be37448ea35e8aded345798ddc918fe1b89e7f90af7c0615902

data/lib/rest-ftp-daemon.rb

@@ -14,12 +14,12 @@ require "sys/cpu"
 require 'rest-ftp-daemon/constants'
 require 'rest-ftp-daemon/config'
 require 'rest-ftp-daemon/exceptions'
-require 'rest-ftp-daemon/common'
 require 'rest-ftp-daemon/helpers'
 require 'rest-ftp-daemon/uri'
 require 'rest-ftp-daemon/job_queue'
 require 'rest-ftp-daemon/worker_pool'
 require 'rest-ftp-daemon/logger'
+require 'rest-ftp-daemon/logger_pool'
 require 'rest-ftp-daemon/job'
 require 'rest-ftp-daemon/notification'
 require 'rest-ftp-daemon/api/root'

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -32,14 +32,18 @@ module RestFtpDaemon
           @only = params["only"].to_sym
         end
 
-        if @only.nil?
-          @jobs_popped = popped_jobs
+        case @only
+        when nil
+          @jobs_current = popped_jobs
+        when :queued
+          @jobs_current = @jobs_queued
         else
-          @jobs_popped = $queue.popped_reverse_sorted_by_status @only
+          @jobs_current = $queue.popped_reverse_sorted_by_status @only
         end
 
-        # Count jobs for each status
+        # Count jobs for each status and total
         @counts = $queue.counts_by_status
+        @count_all = $queue.all_size
 
         # Get workers status
         @gworker_statuses = $pool.get_worker_statuses

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -69,7 +69,8 @@ module RestFtpDaemon
       end
 
       post '/jobs/' do
-        info "POST /jobs #{params.inspect}"
+        info "POST /jobs"
+        # #{params.inspect}"
         # request.body.rewind
         begin
 

data/lib/rest-ftp-daemon/api/root.rb

@@ -5,7 +5,8 @@ module RestFtpDaemon
 
 ####### CLASS CONFIG
 
-      logger RestFtpDaemon::Logger.new(:api, "API")
+      # logger RestFtpDaemon::Logger.new(:api, "API")
+      logger RestFtpDaemon::LoggerPool.instance.get :api
 
       do_not_route_head!
       do_not_route_options!
@@ -16,26 +17,25 @@ module RestFtpDaemon
       format :json
 
 
-####### EXCETPIONS HANDLING
-      # FIXME
-      # rescue_from :all do |e|
-      #   error_response(message: "Internal server error", status: 500)
-      # end
+####### INITIALIZATION
+
+      def initialize
+        super
+      end
 
 
 ####### HELPERS
 
       helpers do
 
-        def info message, level = 0
-          Root.logger.info(message, level)
+        def info message
+          Root.logger.info_with_id message
         end
 
         def api_error exception
           {
           :error => exception.message,
           :message => exception.backtrace.first,
-          #:backtrace => exception.backtrace,
           }
         end
 

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_CONF = "/etc/#{APP_NAME}.yml"
-APP_VER = "0.95.2"
+APP_VER = "0.100"
 
 # Some global constants
 IDENT_JOB_LEN = 4
@@ -12,7 +12,9 @@ IDENT_RANDOM_LEN = 8
 DEFAULT_CONNECT_TIMEOUT_SEC = 30
 DEFAULT_UPDATE_EVERY_KB = 2048
 DEFAULT_WORKERS = 1
-DEFAULT_LOGS_PROGNAME_TRIM = 12
+
+DEFAULT_LOGS_PIPE_WIDTH = 15
+DEFAULT_LOGS_ID_WIDTH = 8
 
 # Initialize markers
 APP_STARTED = Time.now

data/lib/rest-ftp-daemon/helpers.rb

@@ -28,14 +28,18 @@ module RestFtpDaemon
       rand(36**len).to_s(36)
     end
 
-    def self.tokenize(item)
+    def self.tokenize item
       "[#{item}]"
     end
 
-    def self.highlight_tokens(path)
+    def self.highlight_tokens path
       path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
     end
 
+    # def self.hide_password url
+    #   path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
+    # end
+
     def self.extract_filename path
       # match everything that's after a slash at the end of the string
       m = path.match /\/?([^\/]+)$/

data/lib/rest-ftp-daemon/job.rb

@@ -4,7 +4,7 @@ require 'double_bag_ftps'
 require 'timeout'
 
 module RestFtpDaemon
-  class Job < RestFtpDaemon::Common
+  class Job
 
     FIELDS = [:source, :target, :priority, :notify, :overwrite, :mkdir]
 
@@ -26,7 +26,6 @@ module RestFtpDaemon
     def initialize job_id, params={}
       # Call super
       # super()
-      info "Job.initialize"
 
       # Init context
       @id = job_id.to_s
@@ -36,25 +35,27 @@ module RestFtpDaemon
       end
       @params = {}
 
+      # Logger
+      # @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
+      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
+
       # Protect with a mutex
       @mutex = Mutex.new
 
-      # Logger
-      @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
-
       # Flag current job
       @queued_at = Time.now
       @status = :created
 
       # Send first notification
       #info "Job.initialize/notify"
+      info "Job.initialized"
       client_notify "rftpd.queued"
     end
 
     def close
       # Close logger
-      info "Job.close"
-      @logger.close
+      # info "Job.close"
+      #@logger.close
     end
 
     def process
@@ -110,6 +111,9 @@ module RestFtpDaemon
       rescue Errno::EHOSTDOWN => exception
         return oops "rftpd.ended", exception, :job_host_is_down
 
+      rescue Errno::ENOTCONN => exception
+        return oops "rftpd.ended", exception, :job_connexion_failed
+
       rescue Errno::ECONNREFUSED => exception
         return oops "rftpd.ended", exception, :job_connexion_refused
 
@@ -294,7 +298,6 @@ module RestFtpDaemon
       # Check source files presence and compute total size, they should be there, coming from Dir.glob()
       @transfer_total = 0
       source_matches.each do |filename|
-        # @ftp.close
         raise RestFtpDaemon::JobSourceNotFound unless File.exists? filename
         @transfer_total += File.size filename
       end
@@ -312,18 +315,26 @@ module RestFtpDaemon
       $queue.counter_add :transferred, @transfer_total
 
       # Close FTP connexion
+      @ftp.close
       info "Job.transfer disconnecting"
       @status = :disconnecting
-      @ftp.close
     end
 
   private
 
+    def info message, level = 0
+      return if @logger.nil?
+      @logger.info_with_id message, level: level, id: @id
+    end
+
     def oops signal_name, exception, error_name = nil, include_backtrace = false
       # Log this error
       error_name = exception.class if error_name.nil?
       info "Job.oops si[#{signal_name}] er[#{error_name.to_s}] ex[#{exception.class}]"
 
+      # Close ftp connexion if open
+      @ftp.close unless @ftp.nil?
+
       # Update job's internal status
       @status = :failed
       @error = error_name
@@ -346,8 +357,6 @@ module RestFtpDaemon
 
       # Prepare notification if signal given
       return unless signal_name
-
-      # Send the real notification
       client_notify signal_name, error_name, notif_status
     end
 
@@ -515,7 +524,7 @@ module RestFtpDaemon
         stack << (Helpers.format_bytes @transfer_sent, "B")
         stack << (Helpers.format_bytes @transfer_total, "B")
         stack << (Helpers.format_bytes bitrate0, "bps")
-        info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % txt)}.join("\t")
+        info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PIPE_WIDTH.to_i}s" % txt)}.join("\t")
 
         # Update time pointer
         t0 = Time.now
@@ -544,12 +553,20 @@ module RestFtpDaemon
     end
 
     def client_notify signal, error = nil, status = {}
-      RestFtpDaemon::Notification.new @notify, {
-        id: @id,
-        signal: signal,
-        error: error,
-        status: status,
-        }
+      # Skip if no URL given
+      return unless @notify
+
+      # Ok, create a notification!
+      begin
+        RestFtpDaemon::Notification.new @notify, {
+          id: @id,
+          signal: signal,
+          error: error,
+          status: status,
+          }
+      rescue Exception => ex
+        info "Job.client_notify exception: #{ex.inspect}"
+      end
     end
 
     def get_bitrate total, last_timestamp

data/lib/rest-ftp-daemon/job_queue.rb

@@ -7,19 +7,19 @@ module RestFtpDaemon
     attr_reader :popped
 
     def initialize
-      # # Logger
-      @logger = RestFtpDaemon::Logger.new(:queue, "QUEUE")
-
       # Instance variables
       @queued = []
       @popped = []
-
       @waiting = []
       @queued.taint          # enable tainted communication
       @waiting.taint
       self.taint
       @mutex = Mutex.new
 
+      # # Logger
+      # @logger = RestFtpDaemon::Logger.new(:queue, "QUEUE")
+      @logger = RestFtpDaemon::LoggerPool.instance.get :queue
+
       # Identifiers generator
       @last_id = 0
       #@prefix = SecureRandom.hex(IDENT_JOB_LEN)
@@ -110,10 +110,12 @@ module RestFtpDaemon
     def push job
       # Check that item responds to "priorty" method
       raise "JobQueue.push: job should respond to priority method" unless job.respond_to? :priority
+      raise "JobQueue.push: job should respond to id method" unless job.respond_to? :id
 
       @mutex.synchronize do
         # Push job into the queue
         @queued.push job
+        #info "JobQueue.push: #{job.id}"
 
         # Tell the job it's been queued
         job.set_queued if job.respond_to? :set_queued
@@ -132,13 +134,16 @@ module RestFtpDaemon
 
 
     def pop(non_block=false)
+      # info "JobQueue.pop"
       @mutex.synchronize do
         while true
           if @queued.empty?
+            # info "JobQueue.pop: empty"
             raise ThreadError, "queue empty" if non_block
             @waiting.push Thread.current
             @mutex.sleep
           else
+            # info "JobQueue.pop: great, I'm not empty!!"
             return pick_one
           end
         end
@@ -211,10 +216,6 @@ module RestFtpDaemon
 
     end
 
-    def info message, level = 0
-      @logger.add(Logger::INFO, "#{'  '*(level+1)} #{message}", progname) unless @logger.nil?
-    end
-
     def pick_one  # called inside a mutex/sync
       # Sort jobs by priority and get the biggest one
       picked = ordered_queue.last
@@ -225,14 +226,15 @@ module RestFtpDaemon
       @popped.push picked
 
       # Return picked
+      #info "JobQueue.pick_one: #{picked.id}"
       picked
     end
 
-
   private
 
-    def info message, level = 0
-      @logger.info(message, level) unless @logger.nil?
+    def info message
+      return if @logger.nil?
+      @logger.info_with_id message
     end
 
   end

data/lib/rest-ftp-daemon/logger.rb

@@ -1,31 +1,10 @@
-module RestFtpDaemon
-  class Logger
+class Logger
 
-    def initialize context, progname
-      # Init
-      @context = context
-      @progname = progname
-
-      # Compute file path
-      logfile = Settings.logs[@context] if Settings.logs.is_a? Hash
-
-      # Instantiate a logger if it's non-null
-      @logger = ActiveSupport::Logger.new(logfile, 'daily') unless logfile.nil?
-    end
-
-    def info message, level = 0
-      return if @logger.nil?
-
-      stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
-      progname = "%-#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % @progname
-      line = "#{stamp} #{progname} \t#{'  '*(level+1)}#{message}"
-
-      if @logger.nil?
-        puts line
-      else
-        @logger.add(ActiveSupport::Logger::INFO, line)
-      end
-    end
+  attr_accessor :pipe
 
+  def info_with_id message, options = {}
+    field_id = "%#{-DEFAULT_LOGS_ID_WIDTH.to_i}s" % options[:id].to_s
+    add Logger::INFO, "#{field_id} \t#{'  '*(options[:level].to_i+1)}#{message}"
   end
+
 end

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -0,0 +1,37 @@
+require 'singleton'
+require 'logger'
+
+module RestFtpDaemon
+  class LoggerPool
+    include Singleton
+
+    def initialize
+      @loggers = {}
+    end
+
+    def get pipe
+      @loggers[pipe] ||= create(pipe)
+    end
+
+    def create pipe
+      # Compute file path / STDERR
+      logfile = Settings.logs[pipe] if Settings.logs.is_a? Hash
+      logfile ||= STDERR
+      #logfile ||= STDOUT
+
+      # Create the logger and return it
+      logger = Logger.new(logfile, 'daily')   #, 10, 1024000)
+      logger.progname = pipe.to_s.upcase
+      logger.formatter = proc do |severity, datetime, progname, message|
+        # stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+        stamp = datetime.strftime("%Y-%m-%d %H:%M:%S")
+        field_pipe = "%-#{DEFAULT_LOGS_PIPE_WIDTH.to_i}s" % progname
+        "#{stamp} #{field_pipe} #{message}\n"
+      end
+
+      # Finally return this logger
+      logger
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/notification.rb

@@ -1,7 +1,7 @@
 require 'net/http'
 
 module RestFtpDaemon
-  class Notification < RestFtpDaemon::Common
+  class Notification
     attr_accessor :job_id
     attr_accessor :signal
     attr_accessor :error
@@ -9,14 +9,14 @@ module RestFtpDaemon
     attr_accessor :status
     attr_accessor :url
     attr_accessor :job
-    attr_accessor :key
 
     def initialize url, params
       # Generate a random key
-      key = Helpers.identifier(IDENT_NOTIF_LEN)
+      @id = Helpers.identifier(IDENT_NOTIF_LEN)
 
       # Logger
-      @logger = RestFtpDaemon::Logger.new(:workers, "NOTIF #{key}")
+      # @logger = RestFtpDaemon::Logger.new(:workers, "NOTIF #{key}")
+      @logger = RestFtpDaemon::LoggerPool.instance.get :notify
 
       # Check context
       if url.nil?
@@ -59,5 +59,11 @@ module RestFtpDaemon
 
   protected
 
+    def info message, level = 0
+      return if @logger.nil?
+      #puts "JOB: #{message}"
+      @logger.info_with_id message, level: level, id: @id
+    end
+
   end
 end

data/lib/rest-ftp-daemon/static/css/main.css

@@ -3,10 +3,11 @@
   }
 
 .transfer-method {
-  display: inline-block;
-  /*width: 35px;*/
-  padding: 3px 3px;
-  font-weight: normal;
+  width: 35px;
+  }
+
+.worker-label {
+  width: 25px;
   }
 
 .nobr {
@@ -25,20 +26,25 @@ table tr td, .fixed {
   /*font-weight: bold;*/
   }
 
-/*
-.label-outline.badge-success {
-    border-color: #468847;
-}
-*/
+.label {
+  font-size: 90%;
+  padding: 2px 4px;
+  font-weight: normal;
+  display: inline-block;
+  font-family: sans-serif;
+  }
+
 .label-outline {
-    color: black;
-    border: 1px solid #999;
-    background-color: transparent;
-}
+  color: black;
+  border: 1px solid #999;
+  background-color: transparent;
+  }
 
-.flag {
-  margin-right: 3px;
-}
+.label-contrast {
+  color: white;
+  border: 1px dotted #999;
+  background-color: black;
+  }
 
 h2 {
   font-size: 1.5em;
@@ -55,4 +61,3 @@ h2 {
   margin-bottom: 0;
   }
 
-

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -33,7 +33,7 @@
             .btn-group.btn-group-md
               - klass = @only.nil? ? "btn-info" : ""
               %a.btn.btn-default{href: "?only=", class: klass}
-                ALL (#{@jobs_queued.size} + #{@jobs_popped.size})
+                ALL (#{@count_all})
             .btn-group.btn-group-md
               - @counts.each do |status, count|
                 - klass = (status == @only) ? "btn-info" : ""
@@ -59,8 +59,7 @@
               %tr
                 %th{colspan: 10}
 
-            = render :dashboard_jobs, {jobs: @jobs_popped, counts: @counts}
-
+            = render :dashboard_jobs, {jobs: @jobs_current, counts: @counts}
 
 
       .row

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -27,7 +27,7 @@
 
     %td
       - unless job.wid.nil?
-        .label.label-warning.flag= "w#{job.wid}"
+        .label.label-default.flag.worker-label= job.wid
 
     %td{title: job.get(:source_path)}
       =# Helpers.job_method_label job.get(:source_method)
@@ -70,4 +70,3 @@
       - if (bitrate = job.get :transfer_bitrate)
         =  Helpers.format_bytes(bitrate, "bps")
 
-

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -1,11 +1,11 @@
 module RestFtpDaemon
-  class WorkerPool < RestFtpDaemon::Common
+  class WorkerPool
 
     attr_reader :requested, :processed, :wid
 
-    def initialize(number_threads)
+    def initialize number_threads
       # Logger
-      @logger = RestFtpDaemon::Logger.new(:workers, "WORKER")
+      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
 
       # Check parameters
       raise "A thread count of #{number_threads} is less than one" if number_threads < 1
@@ -29,16 +29,6 @@ module RestFtpDaemon
 
     end
 
-    # def wait
-    #   item = @out.pop
-    #   @lock.synchronize { @processed += 1 }
-    #   block_given? ? (yield item) : item
-    # end
-
-    # def progname
-    #   "WORKER #{@wid}"
-    # end
-
     def work wid
       worker_status wid, "starting"
       loop do
@@ -74,8 +64,6 @@ module RestFtpDaemon
         # Clean job status
         worker_status wid, :ready
         job.wid = nil
-        # sleep 1
-
         end
 
       end
@@ -89,6 +77,11 @@ module RestFtpDaemon
 
   protected
 
+    def info message
+      return if @logger.nil?
+      @logger.info_with_id message
+    end
+
     def worker_status wid, status, jobid = nil
       @mutex.synchronize do
         @statuses[wid] ||= {}

data/rest-ftp-daemon.yml.sample

@@ -17,9 +17,11 @@ defaults: &defaults
     #clean_finished: 600
 
   logs:
-    thin:
-    api:
-    workers:
+    thin:    "/tmp/rftpd-environment-thin.log"
+    queue:   "/tmp/rftpd-environment-core.log"
+    api:     "/tmp/rftpd-environment-core.log"
+    workers: "/tmp/rftpd-environment-work.log"
+    notify:  "/tmp/rftpd-environment-work.log"
 
 preprod:
   <<: *defaults

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.95.2
+  version: '0.100'
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-08 00:00:00.000000000 Z
+date: 2014-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -203,7 +203,6 @@ files:
 - lib/rest-ftp-daemon/api/root.rb
 - lib/rest-ftp-daemon/api/routes.rb
 - lib/rest-ftp-daemon/api/status.rb
-- lib/rest-ftp-daemon/common.rb
 - lib/rest-ftp-daemon/config.rb
 - lib/rest-ftp-daemon/constants.rb
 - lib/rest-ftp-daemon/exceptions.rb
@@ -211,6 +210,7 @@ files:
 - lib/rest-ftp-daemon/job.rb
 - lib/rest-ftp-daemon/job_queue.rb
 - lib/rest-ftp-daemon/logger.rb
+- lib/rest-ftp-daemon/logger_pool.rb
 - lib/rest-ftp-daemon/notification.rb
 - lib/rest-ftp-daemon/static/css/bootstrap.css
 - lib/rest-ftp-daemon/static/css/main.css

data/lib/rest-ftp-daemon/common.rb

@@ -1,14 +0,0 @@
-module RestFtpDaemon
-
-  class Common
-
-  protected
-
-    # FIXME: should be moved to class itself to get rid of this parent class
-
-    def info message, level = 0
-      @logger.info(message, level) unless @logger.nil?
-    end
-
-  end
-end