rest-ftp-daemon 0.104.5 → 0.200

data/lib/rest-ftp-daemon/notification.rb

@@ -12,56 +12,81 @@ module RestFtpDaemon
 
     def initialize url, params
       # Generate a random key
-      @id = Helpers.identifier(IDENT_NOTIF_LEN)
+      @id = Helpers.identifier(NOTIFY_IDENTIFIER_LEN)
 
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :notify
 
       # Check context
+
       if url.nil?
-        info "skipping (missing url): #{params}"
+        info "skipping (missing url): #{params.inspect}"
         return
-      elsif params[:signal].nil?
-        info "skipping (missing signal): #{params}"
+
+      elsif params[:event].nil?
+        info "skipping (missing event): #{params.inspect}"
         return
+
       else
-        # info "queuing: s[#{params[:signal]}] e[#{params[:error]}] u[#{url}]"
-        info "queuing: #{params.inspect}"
+        #info "created: OK"
+        # info "created: #{params.class}"
+        info "created #{params.inspect}"
+
       end
 
       # Params
       body = {
         id:       params[:id].to_s,
-        signal:   params[:signal],
+        signal:   "#{NOTIFY_PREFIX}.#{params[:event].to_s}",
         error:    params[:error],
-        host:     Settings['host'].to_s,
+        host:     Settings.host.to_s,
         }
-      body[:status] = params[:status] if (params[:status].is_a? Enumerable) unless params[:status].nil?
+      body[:status] = params[:status] if params[:status].is_a? Enumerable
 
       # Send message in a thread
       Thread.new do |thread|
         # Prepare query
         uri = URI(url)
-        headers = {"Content-Type" => "application/json",
-                  "Accept" => "application/json"}
+        headers = {
+          'Content-Type'  => 'application/json',
+          'Accept'        => 'application/json',
+          'User-Agent'    => "#{APP_NAME} - #{APP_VER}"
+           }
+        data = body.to_json
+        info "sending #{data}"
+
+        # Prepare HTTP client
+        http = Net::HTTP.new uri.host, uri.port
+        # http.initialize_http_header({'User-Agent' => APP_NAME})
 
         # Post notification
-        info "sending: #{body.inspect}"
-        http = Net::HTTP.new(uri.host, uri.port)
-        response = http.post(uri.path, body.to_json, headers)
+        response = http.post uri.path, data, headers
+
+        # Handle server response / multi-lines
+        response_lines = response.body.lines
+
+        if response_lines.size > 1
+          human_size = Helpers.format_bytes(response.body.bytesize, "B")
+          #human_size = 0
+          info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", lines: response_lines
+        else
+          info "received [#{response.code}] #{response.body.strip}"
+        end
 
-        # info "notify reply: #{response.body.strip}"
-        info "reply: #{response.body.strip}"
       end
 
     end
 
   protected
 
-    def info message, level = 0
+    def info message, context = {}
       return if @logger.nil?
-      #puts "JOB: #{message}"
-      @logger.info_with_id message, level: level, id: @id
+
+      # Inject context
+      context[:id] = @id
+      context[:origin] = self.class
+
+      @logger.info_with_id message, context
     end
 
   end

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -5,7 +5,7 @@
     %link{ href:"/css/bootstrap.css" , rel: "stylesheet"}
     %link{ href:"/css/main.css" , rel: "stylesheet"}
     %link{ href:"http://fonts.googleapis.com/css?family=Inconsolata:400,700" , rel: "stylesheet"}
-    %title="#{Settings['host']} [#{Settings.namespace}] #{APP_NAME}"
+    %title="#{Settings.host} [#{Settings.namespace}] #{APP_NAME}"
 
   %body
 
@@ -49,10 +49,9 @@
               %th target
               %th date
               %th status
-              %th progress
               %th error
               %th.text-right size
-              %th.text-right rate
+              %th.text-right bitrate
               %th
 
             - if @only.nil? && !@jobs_queued.empty?

data/lib/rest-ftp-daemon/views/dashboard_headers.haml

@@ -9,7 +9,7 @@
 
 .btn-group.btn-group-sm
   .btn.btn-default.btn-success Host
-  .btn.btn-default= Settings['host']
+  .btn.btn-default= Settings.host
 
 .btn-group.btn-group-sm
   .btn.btn-default.btn-warning Load

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -3,18 +3,18 @@
   - progress = job.get :progress
   - source_count = job.get(:source_count) || 0
   - source_processed = job.get(:source_processed) || 0
-  - source_processing = job.get(:source_processing)
+  - source_current = job.get(:source_current)
   - presented = present job, :with => RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
   - errmsg = job.get :error_message
   - method = job.get(:target_method)
 
   - if !job.error.nil?
     - trclass = "danger"
-  - elsif job.status == :uploading
+  - elsif job.status == JOB_STATUS_UPLOADING
     - trclass = "info"
-  - elsif job.status == :finished
+  - elsif job.status == JOB_STATUS_FINISHED
     - trclass = "success"
-  - elsif job.status == :queued
+  - elsif job.status == JOB_STATUS_QUEUED
     - trclass = "active"
   - else
     - trclass = "warning"
@@ -41,29 +41,27 @@
     %td
       = job.status
 
-      -# if (source_count > 1) && (source_processed < source_count)
       - if (source_processed < source_count)
-        %small= " (#{source_processed}/#{source_count})"
+        = " (#{source_processed}/#{source_count})"
 
-      - unless source_processing.nil?
+      - unless progress.nil?
+        = "#{progress}%"
+
+      - if job.status != JOB_STATUS_FINISHED
         %br
-        %b= " #{source_processing}"
+        %b= source_current unless source_current.nil?
 
     %td
-      - unless progress.nil?
+      - if !job.error && job.status != JOB_STATUS_FINISHED
+        -# unless progress.nil? ||
         .progress
           .progress-bar{style:"width: #{progress}%;"}
-            = Helpers.format_bytes job.get(:transfer_sent), "B"
+            = "#{progress}%" unless progress.nil?
 
-    %td
-      - unless job.error || job.status == :finished
-        - unless progress.nil?
-          = "#{progress}%"
       - else
         .error{title: errmsg}
           = Helpers.text_or_empty(job.error)
 
-
     %td.nobr.text-right
       = Helpers.format_bytes(size, "B")
 
@@ -76,5 +74,4 @@
         .label.label-info.flag.worker-label= job.priority
 
       - unless job.wid.nil?
-        %br
         .label.label-warning.flag.worker-label= job.wid

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -1,3 +1,5 @@
+
+
 module RestFtpDaemon
   class WorkerPool
 
@@ -7,39 +9,57 @@ module RestFtpDaemon
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
 
+
       # Check parameters
       raise "A thread count of #{number_threads} is less than one" if number_threads < 1
 
-      # Prepare status hash
+      # Prepare status hash and vars
       @statuses = {}
       @workers = {}
+      @mutex = Mutex.new
+      @counter = 0
 
       # Create worker threads
       info "WorkerPool initializing with #{number_threads} workers"
-      @mutex = Mutex.new
-      @counter = 0
+      create_worker_threads number_threads
+
+    end
+
+    def worker_vars
+      vars = {}
+
+      @workers.each do |name, thread|
+        #currents[thread.id] = thread.current
+        vars[thread[:name]] = thread[:vars]
+      end
+
+      vars
+    end
+
+  private
 
-      number_threads.times do
+    def create_worker_threads n
+      n.times do
         # Increment counter
         @mutex.synchronize do
           @counter +=1
         end
-        name = "w#{@counter}"
 
-        th = Thread.new name do
+        # Create a dedicated thread for this worker
+        name = "w#{@counter}"
+        @workers[name] = create_worker_thread name
+      end
 
-          # Set thread context
-          Thread.current[:name] = name
-          Thread.current[:vars] = {
-            started_at: Time.now,
-            }
+    end
+    def create_worker_thread name
+              @workers[name] = Thread.new name do
 
-          # Start working
-          work
-        end
+        # Set thread context
+        Thread.current[:name] = name
+        Thread.current[:vars] = { started_at: Time.now }
 
-        # Add this worker to the ThreadGroup
-        @workers[name] = th
+        # Start working
+        work
       end
 
     end
@@ -70,12 +90,7 @@ module RestFtpDaemon
           $queue.counter_inc :jobs_processed
 
         rescue Exception => ex
-          worker_status :crashed
-          info "UNHANDLED EXCEPTION: #{ex.message}"
-          ex.backtrace.each do |line|
-            info line, 1
-          end
-          sleep 1
+          handle_job_uncaught_exception job, ex
 
         else
           # Clean job status
@@ -87,25 +102,42 @@ module RestFtpDaemon
       end
     end
 
-    def worker_vars
-      vars = {}
+    def handle_job_uncaught_exception job, ex
+      begin
+        # Log the exception
+        info "UNHDNALED EXCEPTION: job: #{ex.message}", lines: ex.backtrace
+
+        # Tell the worker has creashed
+        worker_status :crashed
+
+        # Flag the job as crashed
+        job.oops_after_crash ex
+
+      rescue Exception => ex
+        info "DOUBLE EXCEPTION: #{ex.message}", lines: ex.backtrace
 
-      @workers.each do |name, thread|
-        #currents[thread.id] = thread.current
-        vars[thread[:name]] = thread[:vars]
       end
 
-      vars
+      # Wait a bit
+      sleep 1
     end
 
+
   protected
 
     def ping
     end
 
-    def info message
+    def info message, context = {}
       return if @logger.nil?
-      @logger.info_with_id message, id: Thread.current[:name]
+
+      # Ensure context is a hash of options and inject context
+      context = {} unless context.is_a? Hash
+      context[:id] = Thread.current[:name]
+      context[:origin] = self.class
+
+      # Forward to logger
+      @logger.info_with_id message, context
     end
 
     def worker_status status, jobid = nil

data/rest-ftp-daemon.gemspec

@@ -35,6 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "double-bag-ftps"
   spec.add_runtime_dependency "facter"
   spec.add_runtime_dependency "sys-cpu"
-  #spec.add_runtime_dependency "socket"
   spec.add_runtime_dependency "timeout"
 end

data/rest-ftp-daemon.yml.sample

@@ -21,6 +21,9 @@ defaults: &defaults
     #clean_failed: 3600
     #clean_finished: 600
 
+  debug:
+    ftp: false
+
   logs:
     thin:    "/var/log/rftpd-environment-thin.log"
     queue:   "/var/log/rftpd-environment-core.log"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.104.5
+  version: '0.200'
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-18 00:00:00.000000000 Z
+date: 2015-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler