rest-ftp-daemon 0.103.1 → 0.104.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c059a568b0a22503a734c966807a19c8ff426333
-  data.tar.gz: a3d2443af653af100e642e86ba7144a2b9985495
+  metadata.gz: 3f80c711efa90f45d67c8b2c7648b4a951b8da6b
+  data.tar.gz: 9dbda28ce22feaff2becdc4d0f211963569134f7
 SHA512:
-  metadata.gz: 07afa716bc8b3b701377cfe1a9452e22238cc8e82af287f544e499cc2a4d263bd266e3d83f8a2aa53ca37381653d1fff961bdbff6bf9446167cf1276fb8bddf8
-  data.tar.gz: 31fb4ea27d1e64278563eb3e2e93375c24d1f9794e632acd7afbeb897dc0a6d88024fe2dd3c4393648fa151b1f54c35432323f45cd90b4f9d368feeb208078a1
+  metadata.gz: 4bca8882c3ad0b481f7ce99e9588d6ceba5a6ab009b60f74bfbaea0bc40536e8789753527a3cfb89515bf53de4e8314c8b57bea65a8caafef38c3b116e867855
+  data.tar.gz: 3971446df3aa2b11007997dd9a78a094787c6ecf55e5f86102c1914c0eb48d72643ff9d3e1074a7a13fff39fa3ef70d76ccf2b8854c086d7d8ca484840438af9

data/bin/rest-ftp-daemon

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-# Try to load libs and constants
+# Try to load external libs
 app_root = File.dirname(__FILE__) + '/../'
 begin
   require "thin"
@@ -10,14 +10,19 @@ begin
 rescue LoadError
   raise "EXITING: some of basic libs were not found: thin, optparse, socket, timeout"
 end
-require File.expand_path("#{app_root}/lib/rest-ftp-daemon/constants")
+
+# Load helpers and constants
+[:constants, :helpers].each do |lib|
+  require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}")
+end
+
 puts
 #puts "--- #{APP_NAME} #{APP_VER}"
 
 # Detect options from ARGV
 options = {}
 parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{File.basename $0} [options] start|stop|restart"
+  opts.banner = "Usage: #{File.basename $0} [options] start|stop"
   opts.on("-c", "--config CONFIGFILE")                                 { |config| APP_CONF = config }
   opts.on("-e", "--environment ENV")                                   { |env| APP_ENV = env }
   opts.on("",   "--dev")                                               { APP_ENV = "development" }
@@ -34,7 +39,7 @@ end
 begin
   parser.order!(ARGV)
   command = ARGV.shift
-  unless ["start", "stop", "restart"].include? command
+  unless ["start", "stop"].include? command
     puts parser
     exit 11
   end
@@ -51,10 +56,14 @@ unless File.exists? APP_CONF
 end
 
 # Load helpers and config, and merge options from ARGV into settings
-[:helpers, :config].each do |lib|
-  require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}")
+begin
+  require File.expand_path("#{app_root}/lib/rest-ftp-daemon/config")
+  Settings.merge!(options)
+rescue Psych::SyntaxError => e
+  puts "EXITING: config file syntax error: #{e.message}"
+  exit 16
 end
-Settings.merge!(options)
+
 
 # Display compiled configuration
 puts "--- #{APP_NAME} #{APP_VER}"

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -46,7 +46,7 @@ module RestFtpDaemon
         @count_all = $queue.all_size
 
         # Get workers status
-        @gworker_statuses = $pool.get_worker_statuses
+        @worker_vars = $pool.worker_vars
 
         # Compile haml template
         output = render :dashboard

data/lib/rest-ftp-daemon/api/status.rb

@@ -17,6 +17,7 @@ module RestFtpDaemon
           uptime: (Time.now - APP_STARTED).round(1),
           counters: $queue.counters,
           status: $queue.counts_by_status,
+          vars: $pool.worker_vars,
           jobs_count: $queue.all_size,
           jobs_queued: $queue.queued.collect(&:id),
           jobs_popped: $queue.popped.collect(&:id),

data/lib/rest-ftp-daemon/constants.rb

@@ -1,13 +1,13 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
-APP_VER = "0.103.1"
+APP_VER = "0.104.5"
 
 # Some global constants
 IDENT_JOB_LEN = 4
 IDENT_NOTIF_LEN = 4
 IDENT_RANDOM_LEN = 8
 IDENT_TEMPFILE_LEN = 8
-DEFAULT_LOGS_PIPE_LEN = 15
+DEFAULT_LOGS_PIPE_LEN = 12
 DEFAULT_LOGS_ID_LEN = 8
 
 # Some defaults

data/lib/rest-ftp-daemon/helpers.rb

@@ -83,26 +83,31 @@ module RestFtpDaemon
       when :file
         'label-primary'
       when :ftp
-        'label-danger'
-      when :ftps
-        'label-warning'
-      when :ftpes
         'label-warning'
+      when :ftps, :ftpes
+        'label-success'
       else
-         'label-default'
+        'label-default'
       end
       "<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
     end
 
     # Dates and times: date with time generator
-    def self.datetime_full param
-      return "-"  if param.nil?
-      return param.to_datetime.strftime("%d.%m.%Y %H:%M")
+    def self.datetime_full datetime
+      return "-"  if datetime.nil?
+      return datetime.to_datetime.strftime("%d.%m.%Y %H:%M")
+    end
+
+    def self.datetime_short datetime
+      # return param.class
+      return "-" if datetime.nil?
+      return "?" unless datetime.respond_to? :to_date
+      return datetime.to_datetime.strftime("%H:%M") if datetime.to_date == Time.now.to_date
+      return datetime.to_datetime.strftime("%d/%m %H:%M")
     end
 
-    def self.datetime_short param
-      return "-"  if param.nil?
-      return param.to_datetime.strftime("%d/%m %H:%M")
+    def self.hide_credentials_from_url url
+      url.sub(/([a-z]+:\/\/[^\/]+):[^\/]+\@/, '\1@' )
     end
 
   end

data/lib/rest-ftp-daemon/job.rb

@@ -78,9 +78,6 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobMissingAttribute => exception
         return oops "rftpd.started", exception, :missing_attribute
 
-      # rescue RestFtpDaemon::JobSourceNotFound => exception
-      #   return oops "rftpd.started", exception, :job_source_not_found
-
       rescue RestFtpDaemon::JobUnresolvedTokens => exception
         return oops "rftpd.started", exception, :unresolved_tokens
 
@@ -97,7 +94,7 @@ module RestFtpDaemon
         return oops "rftpd.started", exception, :assertion_failed
 
       rescue RestFtpDaemon::RestFtpDaemonException => exception
-        return oops "rftpd.started", exception, :prepare_failed
+        return oops "rftpd.started", exception, :prepare_failed, true
 
       rescue Exception => exception
         return oops "rftpd.started", exception, :prepare_unhandled, true
@@ -134,7 +131,10 @@ module RestFtpDaemon
         return oops "rftpd.ended", exception, :conn_openssl_error
 
       rescue Net::FTPPermError => exception
-        return oops "rftpd.ended", exception, :perm_error
+        return oops "rftpd.ended", exception, :ftp_perm_error
+
+      rescue Net::FTPTempError => exception
+        return oops "rftpd.ended", exception, :ftp_temp_error
 
       rescue Errno::EMFILE => exception
         return oops "rftpd.ended", exception, :too_many_open_files
@@ -152,7 +152,7 @@ module RestFtpDaemon
         return oops "rftpd.started", exception, :assertion_failed
 
       rescue RestFtpDaemon::RestFtpDaemonException => exception
-        return oops "rftpd.ended", exception, :transfer_failed
+        return oops "rftpd.ended", exception, :transfer_failed, true
 
       rescue Exception => exception
         return oops "rftpd.ended", exception, :transfer_unhandled, true
@@ -192,7 +192,6 @@ module RestFtpDaemon
     def set attribute, value
       @mutex.synchronize do
         @params || {}
-        # return unless @params.is_a? Enumerable
         @updated_at = Time.now
         @params[attribute] = value
       end
@@ -271,7 +270,8 @@ module RestFtpDaemon
 
       # Method assertions and init
       @status = :checking_source
-      raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
+      raise RestFtpDaemon::JobAssertionFailed, "transfer/1" unless @source_path
+      raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @target_url
       @transfer_sent = 0
       set :source_processed, 0
 
@@ -337,7 +337,7 @@ module RestFtpDaemon
       info "Job.oops si[#{signal_name}] er[#{error_name.to_s}] ex[#{exception.class}] #{exception.message}"
 
       # Close ftp connexion if open
-      @ftp.close unless @ftp.welcome.nil?
+      @ftp.close unless @ftp.nil? || @ftp.welcome.nil?
 
       # Update job's internal status
       @status = :failed
@@ -379,7 +379,8 @@ module RestFtpDaemon
       # Method assertions
       info "Job.ftp_init asserts"
       @status = :ftp_init
-      raise RestFtpDaemon::JobAssertionFailed if @target_method.nil? || @target_url.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_init/1" if @target_method.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_init/2" if @target_url.nil?
 
       info "Job.ftp_init target_method [#{@target_method}]"
       case @target_method
@@ -403,23 +404,27 @@ module RestFtpDaemon
       @status = :ftp_connect
       host = @target_url.host
       info "Job.ftp_connect [#{host}]"
-      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/1" if @ftp.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/2" if @target_url.nil?
 
       @ftp.connect(host)
     end
 
     def ftp_login
       @status = :ftp_login
-      info "Job.ftp_login [#{@target_url.user}]"
-      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.user.nil? || @target_url.password.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_login/1" if @ftp.nil?
+
+      # use "anonymous" if user is empty
+      login = @target_url.user || "anonymous"
+      info "Job.ftp_login [#{login}]"
 
-      @ftp.login @target_url.user, @target_url.password
+      @ftp.login login, @target_url.password
     end
 
     def ftp_chdir_or_buildpath path
       # Method assertions
       info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
-      raise RestFtpDaemon::JobAssertionFailed if path.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_chdir_or_buildpath/1" if path.nil?
       @status = :ftp_chdir
 
       # Extract directory from path
@@ -473,7 +478,8 @@ module RestFtpDaemon
 
       # Method assertions
       @status = :ftp_presence
-      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/1" if @ftp.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/2" if @target_url.nil?
 
       # Get file list, sometimes the response can be an empty value
       results = @ftp.list(target_name) rescue nil
@@ -487,8 +493,8 @@ module RestFtpDaemon
     def ftp_transfer source_match, target_name = nil
       # Method assertions
       info "Job.ftp_transfer source_match [#{source_match}]"
-      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil?
-      raise RestFtpDaemon::JobAssertionFailed if source_match.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @ftp.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source_match.nil?
 
       # Use source filename if target path provided none (typically with multiple sources)
       target_name ||= Helpers.extract_filename source_match
@@ -552,8 +558,6 @@ module RestFtpDaemon
         t0 = Time.now
 
         # Notify if requested
-        @status = :uploaded
-        info "Job.ftp_transfer uploaded"
         unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
           notif_status = {
             progress: percent1,

data/lib/rest-ftp-daemon/job_queue.rb

@@ -168,7 +168,8 @@ module RestFtpDaemon
 
     def ordered_popped
       @mutex_counters.synchronize do
-        @popped.sort_by { |item| [item.updated_at] }
+        @popped.sort_by { |item| [ item.wid.to_s, item.updated_at] }
+        # @popped.sort_by { |item| [item.status.to_s, item.wid.to_s, item.updated_at, - item.id.to_i] }
       end
     end
 

data/lib/rest-ftp-daemon/static/css/main.css

@@ -3,11 +3,11 @@
   }
 
 .transfer-method {
-  width: 35px;
+  /*width: 35px;*/
   }
 
 .worker-label {
-  width: 25px;
+  /*width: 25px;*/
   }
 
 .nobr {
@@ -27,8 +27,9 @@ table tr td, .fixed {
   }
 
 .label {
-  font-size: 90%;
-  padding: 2px 4px;
+  font-size: 1em;
+  padding: 3px 3px 1px 3px;
+  margin-bottom: 1px  ;
   font-weight: normal;
   display: inline-block;
   font-family: sans-serif;

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -10,7 +10,7 @@
   %body
 
     .navbar.navbar-default.navbar-fixed
-      .container
+      .container-fluid
 
         .navbar-header
           %h1
@@ -20,7 +20,7 @@
               = "[#{Settings.namespace}]"
 
 
-    .container
+    .container-fluid
 
       = render :dashboard_headers
 
@@ -43,22 +43,22 @@
           %table.table.table-striped.table-hover.table-condensed
             %tr
               %th ID
-              %th P
-              %th W
               %th label
               %th source
+              %th <=>
               %th target
-              %th queued
+              %th date
               %th status
               %th progress
               %th error
               %th.text-right size
-              %th.text-right bitrate
+              %th.text-right rate
+              %th
 
             - if @only.nil? && !@jobs_queued.empty?
               = render :dashboard_jobs, {jobs: @jobs_queued, counts: @counts}
               %tr
-                %th{colspan: 10}
+                %th{colspan: 12}
 
             = render :dashboard_jobs, {jobs: @jobs_current, counts: @counts}
 

data/lib/rest-ftp-daemon/views/dashboard_counters.haml

@@ -3,7 +3,7 @@
 %table.table.table-striped.table-hover.table-condensed
 
   %tr
-    %th counter
+    %th name
     %th.text-right value
 
   - $queue.counters.each do |name, value|

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -6,6 +6,7 @@
   - source_processing = job.get(:source_processing)
   - presented = present job, :with => RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
   - errmsg = job.get :error_message
+  - method = job.get(:target_method)
 
   - if !job.error.nil?
     - trclass = "danger"
@@ -13,30 +14,25 @@
     - trclass = "info"
   - elsif job.status == :finished
     - trclass = "success"
+  - elsif job.status == :queued
+    - trclass = "active"
   - else
     - trclass = "warning"
 
   %tr{class: trclass}
 
-    %td{title: presented.to_json}= job.id
-
-    %td
-      - unless job.priority.nil?
-        = job.priority
-
-    %td
-      - unless job.wid.nil?
-        .label.label-default.flag.worker-label= job.wid
+    %td{title: presented.to_json}
+      %b= job.id
 
     %td= job.label
 
     %td{title: job.get(:source_path)}
-      =# Helpers.job_method_label job.get(:source_method)
       = Helpers.highlight_tokens job.source
 
+    %td
+      = Helpers.job_method_label method
+
     %td{title: job.get(:target_url)}
-      = Helpers.job_method_label job.get(:target_method)
-      =# job.target
       = Helpers.highlight_tokens job.target
 
     %td
@@ -44,8 +40,14 @@
 
     %td
       = job.status
-      - if (source_count > 1) && (source_processed < source_count)
-        %small= " #{source_processed}/#{source_count}"
+
+      -# if (source_count > 1) && (source_processed < source_count)
+      - if (source_processed < source_count)
+        %small= " (#{source_processed}/#{source_count})"
+
+      - unless source_processing.nil?
+        %br
+        %b= " #{source_processing}"
 
     %td
       - unless progress.nil?
@@ -57,8 +59,6 @@
       - unless job.error || job.status == :finished
         - unless progress.nil?
           = "#{progress}%"
-        - unless source_processing.nil?
-          %small= " (#{source_processing})"
       - else
         .error{title: errmsg}
           = Helpers.text_or_empty(job.error)
@@ -71,3 +71,10 @@
       - if (bitrate = job.get :transfer_bitrate)
         =  Helpers.format_bytes(bitrate, "bps")
 
+    %td
+      - unless job.priority.nil?
+        .label.label-info.flag.worker-label= job.priority
+
+      - unless job.wid.nil?
+        %br
+        .label.label-warning.flag.worker-label= job.wid

data/lib/rest-ftp-daemon/views/dashboard_tokens.haml

@@ -8,7 +8,5 @@
 
   - tokens.each do |token, value|
     %tr
-      %td
-        = Helpers.tokenize(token)
-      %td
-        = value
+      %td= Helpers.tokenize token
+      %td= Helpers.hide_credentials_from_url value

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -8,21 +8,21 @@
     %th ID
     %th status
     %th job
-    %th.text-right idle
+    %th.text-right updated
 
-  - @gworker_statuses.each do |wid, infos|
-    - status = infos[:status]
+  - @worker_vars.each do |wid, vars|
+    -# status = vars[:status]
     - style = styles[status] || ""
 
-    - if infos[:active_at].is_a? Time
-      - no_news_for = (Time.now - infos[:active_at]).round(0)
+    - if vars[:updted_at].is_a? Time
+      - no_news_for = (Time.now - vars[:updted_at]).round(0)
     - else
       - no_news_for = "?"
 
     %tr{class: style.to_s}
       %td= wid
-      %td= infos[:status]
-      %td= infos[:jobid]
+      %td= vars[:status]
+      %td= vars[:jobid]
       %td.text-right
         = no_news_for
         s

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -12,84 +12,106 @@ module RestFtpDaemon
 
       # Prepare status hash
       @statuses = {}
+      @workers = {}
 
       # Create worker threads
       info "WorkerPool initializing with #{number_threads} workers"
       @mutex = Mutex.new
       @counter = 0
 
-      for wid in 1..number_threads
-        Thread.new() do
-          @mutex.synchronize do
-            @counter +=1
-          end
-          work("w#{@counter}")
+      number_threads.times do
+        # Increment counter
+        @mutex.synchronize do
+          @counter +=1
+        end
+        name = "w#{@counter}"
+
+        th = Thread.new name do
+
+          # Set thread context
+          Thread.current[:name] = name
+          Thread.current[:vars] = {
+            started_at: Time.now,
+            }
+
+          # Start working
+          work
         end
+
+        # Add this worker to the ThreadGroup
+        @workers[name] = th
       end
 
     end
 
-    def work wid
-      worker_status wid, "starting"
+    def work
+      worker_status :starting
+
       loop do
 
         begin
-          info "worker [#{wid}] waiting for a job"
+          info "waiting for a job"
 
           # Wait for a job to come into the queue
-          worker_status wid, :waiting
+          worker_status :waiting
           job = $queue.pop
 
           # Do the job
-          info "worker [#{wid}] processing [#{job.id}]"
-          worker_status wid, :processing, job.id
-          job.wid = wid
+          info "processing [#{job.id}]"
+
+          worker_status :processing, job.id
+          job.wid = Thread.current[:name]
           job.process
-          info "worker [#{wid}] processed [#{job.id}]"
-          # job.close
-          worker_status wid, :done
+          info "processed [#{job.id}]"
+          job.wid = nil
+          worker_status :done
 
           # Increment total processed jobs count
           $queue.counter_inc :jobs_processed
 
         rescue Exception => ex
-          worker_status wid, :crashed
+          worker_status :crashed
           info "UNHANDLED EXCEPTION: #{ex.message}"
           ex.backtrace.each do |line|
             info line, 1
           end
           sleep 1
+
         else
+          # Clean job status
+          worker_status :free
+          job.wid = nil
 
-        # Clean job status
-        worker_status wid, :ready
-        job.wid = nil
         end
 
       end
     end
 
-    def get_worker_statuses
-      @mutex.synchronize do
-        @statuses
+    def worker_vars
+      vars = {}
+
+      @workers.each do |name, thread|
+        #currents[thread.id] = thread.current
+        vars[thread[:name]] = thread[:vars]
       end
+
+      vars
     end
 
   protected
 
+    def ping
+    end
+
     def info message
       return if @logger.nil?
-      @logger.info_with_id message
+      @logger.info_with_id message, id: Thread.current[:name]
     end
 
-    def worker_status wid, status, jobid = nil
-      @mutex.synchronize do
-        @statuses[wid] ||= {}
-        @statuses[wid][:status] = status
-        @statuses[wid][:jobid] = jobid
-        @statuses[wid][:active_at] = Time.now
-      end
-
+    def worker_status status, jobid = nil
+      Thread.current[:vars][:status] = status
+      Thread.current[:vars][:jobid] = jobid
+      Thread.current[:vars][:updted_at] = Time.now
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.103.1
+  version: 0.104.5
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-03 00:00:00.000000000 Z
+date: 2015-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler