rest-ftp-daemon 0.202.2 → 0.210.0

data/lib/rest-ftp-daemon/job_queue.rb

@@ -3,15 +3,18 @@ require 'securerandom'
 
 module RestFtpDaemon
   class JobQueue < Queue
-    attr_reader :queued
-    attr_reader :popped
+    # attr_reader :queued
+    # attr_reader :popped
+
+    attr_reader :queue
+    attr_reader :jobs
 
     def initialize
       # Instance variables
-      @queued = []
-      @popped = []
+      @queue = []
+      @jobs = []
       @waiting = []
-      @queued.taint          # enable tainted communication
+      @queue.taint          # enable tainted communication
       @waiting.taint
       self.taint
       @mutex = Mutex.new
@@ -47,9 +50,11 @@ module RestFtpDaemon
     end
 
     def generate_id
-      rand(36**8).to_s(36)
-      @last_id ||= 0
-      @last_id += 1
+      # rand(36**8).to_s(36)
+      @mutex.synchronize do
+        @last_id ||= 0
+        @last_id += 1
+      end
       prefixed_id @last_id
     end
 
@@ -76,33 +81,38 @@ module RestFtpDaemon
       end
     end
 
-    def sorted_by_status status
-      # Just use the base if filter is empty
+    def filter_jobs status
+      # No status filter: return all execept queued
       if status.empty?
-        elements = all
+        @jobs.reject { |job| job.status == JOB_STATUS_QUEUED }
+
+      # Status filtering: only those jobs
       else
-        elements = all.select { |item| item.status == status.to_sym }
-      end
+        @jobs.select { |job| job.status == status.to_sym }
 
-      # Sort these elements
-      elements.sort_by do |item|
-        w = JOB_WEIGHTS[item.status] || 0
-        [ w,  item.wid.to_s, item.updated_at.to_s]
       end
-
     end
 
     def counts_by_status
       statuses = {}
-      all.group_by { |job| job.status }.map { |status, jobs| statuses[status] = jobs.size }
+      @jobs.group_by { |job| job.status }.map { |status, jobs| statuses[status] = jobs.size }
       statuses
     end
 
-    def all
-      @queued + @popped
+    def jobs # change for accessor
+      @jobs
+    end
+
+    def jobs_count
+      @jobs.length
     end
-    def all_size
-      @queued.length + @popped.length
+
+    def queued_ids
+      @queue.collect(&:id)
+    end
+
+    def jobs_ids
+      @jobs.collect(&:id)
     end
 
     def find_by_id id, prefixed = false
@@ -110,8 +120,8 @@ module RestFtpDaemon
       id = prefixed_id(id) if prefixed
       info "find_by_id (#{id}, #{prefixed}) > #{id}"
 
-      # Search in both queues
-      @queued.select { |item| item.id == id }.last || @popped.select { |item| item.id == id }.last
+      # Search in jobs queues
+      @jobs.select { |item| item.id == id }.last
     end
 
     def push job
@@ -121,11 +131,17 @@ module RestFtpDaemon
 
       @mutex.synchronize do
         # Push job into the queue
-        @queued.push job
+        @queue.push job
+
+        # Store the job into the global jobs list
+        @jobs.push job
 
-        # Tell the job it's been queued
+        # Inform the job that it's been queued
         job.set_queued if job.respond_to? :set_queued
 
+        # Refresh queue order
+        sort_queue!
+
         # Try to wake a worker up
         begin
           t = @waiting.shift
@@ -139,17 +155,19 @@ module RestFtpDaemon
     alias enq push
 
     def pop(non_block=false)
-      # info "JobQueue.pop"
       @mutex.synchronize do
         while true
-          if @queued.empty?
+          if @queue.empty?
             # info "JobQueue.pop: empty"
             raise ThreadError, "queue empty" if non_block
             @waiting.push Thread.current
             @mutex.sleep
           else
-            # info "JobQueue.pop: great, I'm not empty!!"
-            return pick_one
+            # Refresh queue order
+            # sort_queue!
+
+            # Extract the heaviest item in the queue
+            return @queue.pop
           end
         end
       end
@@ -158,99 +176,89 @@ module RestFtpDaemon
     alias deq pop
 
     def empty?
-      @queued.empty?
+      @queue.empty?
     end
 
     def clear
-      @queued.clear
+      @queue.clear
     end
 
     def num_waiting
       @waiting.size
     end
 
-    def ordered_queue
-      @mutex_counters.synchronize do
-        @queued.sort_by { |item| [item.priority.to_i, - item.id.to_i] }
-      end
-    end
-
-    # def ordered_popped
-    #   @mutex_counters.synchronize do
-    #     @popped.sort_by { |item| [ item.wid.to_s, item.updated_at] }
-    #     # @popped.sort_by { |item| [item.status.to_s, item.wid.to_s, item.updated_at, - item.id.to_i] }
-    #   end
-    # end
-
   protected
 
     def prefixed_id id
       "#{@prefix}.#{id}"
     end
 
+    def sort_queue!
+      @mutex_counters.synchronize do
+        @queue.sort_by! &:weight
+      end
+    end
+
     def conchita_loop
       info "conchita starting with: #{@conchita.inspect}"
       loop do
-        # Do the cleanup
-        conchita_clean JOB_STATUS_FINISHED
-        conchita_clean :failed
+        # Do the cleanup locking the queues
+        # info "conchita: cleanup expired jobs"
+        @mutex.synchronize do
+          conchita_clean JOB_STATUS_FINISHED
+          conchita_clean JOB_STATUS_FAILED
+          conchita_clean JOB_STATUS_QUEUED
+        end
         sleep @conchita[:timer]
       end
     end
 
-    def conchita_clean status
+    def conchita_clean status     # FIXME: clean both @jobs and @queue
       # Init
       return if status.nil?
-      key = "clean_#{status.to_s}"
 
       # Read config state
-      max_age = @conchita[key.to_s]
-      return if [nil, false].include? max_age
+      maxage = @conchita["clean_#{status.to_s}"] || 0
+      #info "conchita_clean status[#{status.to_s}] \t maxage[#{maxage}] s"
+      return unless maxage > 0
 
       # Delete jobs from the queue if their status is (status)
-      @popped.delete_if do |job|
-        # Skip it if wrong status
+      @jobs.delete_if do |job|
+
+        # Skip if wrong status
         next unless job.status == status.to_sym
 
-        # Skip it if updated_at invalid
-        updated_at = job.updated_at
-        next if updated_at.nil?
+        # Skip if updated_at invalid
+        next if job.updated_at.nil?
 
-        # Skip it if not aged enough yet
-        age = Time.now - updated_at
-        next if age < max_age
+        # Skip if not aged enough yet
+        age = Time.now - job.updated_at
+        next if age < maxage
 
         # Ok, we have to clean it up ..
-        info "conchita_clean #{status.inspect} max_age:#{max_age} job:#{job.id} age:#{age}"
+        info "conchita_clean status[#{status.to_s}] maxage[#{maxage}] job[#{job.id}] age[#{age}]"
+
+        # Remove it from the queue if present
+        job_in_queue = @queue.delete job
+        info "   removed queued job [#{job.id}]" unless job_in_queue.nil?
+
+        # Accept to delete it from @jobs
         true
       end
 
     end
 
-    def pick_one  # called inside a mutex/sync
-      # Sort jobs by priority and get the biggest one
-      picked = ordered_queue.last
-      return nil if picked.nil?
-
-      # Move it away from the queue to the @popped array
-      @queued.delete_if { |item| item == picked }
-      @popped.push picked
-
-      # Return picked
-      #info "JobQueue.pick_one: #{picked.id}"
-      picked
-    end
 
   private
 
-    def info message, context = {}
+    def info message, lines = []
       return if @logger.nil?
 
-      # Inject context
-      context[:origin] = self.class
-
       # Forward to logger
-      @logger.info_with_id message, context
+      @logger.info_with_id message,
+        id: @id,
+        lines: lines,
+        origin: self.class.to_s
     end
 
   end

data/lib/rest-ftp-daemon/logger.rb

@@ -9,8 +9,10 @@ class Logger
     context[:level] ||= 0
 
     # Common message header
-    field_id = "%#{-DEFAULT_LOGS_ID_LEN.to_i}s" % context[:id].to_s
-    prefix = "#{field_id}\t#{'  '*(context[:level].to_i+1)}"
+    field_wid = "%#{-LOG_COL_WID.to_i}s" % context[:wid].to_s
+    field_jid = "%#{-LOG_COL_JID.to_i}s" % context[:jid].to_s
+    field_id = "%#{-LOG_COL_ID.to_i}s" % context[:id].to_s
+    prefix = "#{field_wid} \t#{field_jid} \t#{field_id}\t#{'  '*(context[:level].to_i+1)}"
 
     # Send main message
     add Logger::INFO, prefix + message.to_s
@@ -19,7 +21,7 @@ class Logger
     context[:lines].each do |line|
       line.strip!
       next if line.empty?
-      add Logger::INFO, prefix + '   | ' + line[0..DEFAULT_LOGS_TRIM_LINE]
+      add Logger::INFO, prefix + '   | ' + line[0..LOG_TRIM_LINE]
     end if context[:lines].is_a? Enumerable
 
   end

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -21,10 +21,11 @@ module RestFtpDaemon
       # Create the logger and return it
       logger = Logger.new(logfile, 'daily')   #, 10, 1024000)
       logger.progname = pipe.to_s.upcase
+
+      # And the formatter
       logger.formatter = proc do |severity, datetime, progname, message|
-        # stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
         stamp = datetime.strftime("%Y-%m-%d %H:%M:%S")
-        field_pipe = "%-#{DEFAULT_LOGS_PIPE_LEN.to_i}s" % progname
+        field_pipe = "%-#{LOG_PIPE_LEN.to_i}s" % progname
         "#{stamp}\t#{field_pipe}\t#{message}\n"
       end
 

data/lib/rest-ftp-daemon/notification.rb

@@ -1,5 +1,3 @@
-require 'net/http'
-
 module RestFtpDaemon
   class Notification
     attr_accessor :job_id
@@ -13,28 +11,21 @@ module RestFtpDaemon
     def initialize url, params
       # Generate a random key
       @id = Helpers.identifier(NOTIFY_IDENTIFIER_LEN)
+      @jid = nil
 
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :notify
 
       # Check context
-
       if url.nil?
         info "skipping (missing url): #{params.inspect}"
         return
-
       elsif params[:event].nil?
         info "skipping (missing event): #{params.inspect}"
         return
-
-      else
-        #info "created: OK"
-        # info "created: #{params.class}"
-        info "created #{params.inspect}"
-
       end
 
-      # Params
+      # Build body and extract job ID if provided
       body = {
         id:       params[:id].to_s,
         signal:   "#{NOTIFY_PREFIX}.#{params[:event].to_s}",
@@ -42,6 +33,9 @@ module RestFtpDaemon
         host:     Settings.host.to_s,
         }
       body[:status] = params[:status] if params[:status].is_a? Enumerable
+      @jid = params[:id]
+      info "initialized"
+
 
       # Send message in a thread
       Thread.new do |thread|
@@ -55,6 +49,7 @@ module RestFtpDaemon
         data = body.to_json
         info "sending #{data}"
 
+
         # Prepare HTTP client
         http = Net::HTTP.new uri.host, uri.port
         # http.initialize_http_header({'User-Agent' => APP_NAME})
@@ -68,7 +63,7 @@ module RestFtpDaemon
         if response_lines.size > 1
           human_size = Helpers.format_bytes(response.body.bytesize, "B")
           #human_size = 0
-          info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", lines: response_lines
+          info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
         else
           info "received [#{response.code}] #{response.body.strip}"
         end
@@ -79,14 +74,14 @@ module RestFtpDaemon
 
   protected
 
-    def info message, context = {}
+    def info message, lines = []
       return if @logger.nil?
 
-      # Inject context
-      context[:id] = @id
-      context[:origin] = self.class
-
-      @logger.info_with_id message, context
+      @logger.info_with_id message,
+        id: @id,
+        jid: @jid,
+        lines: lines,
+        origin: self.class.to_s
     end
 
   end

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -26,7 +26,7 @@
 
       .row
         #box-jobs.col-md-12
-          = render :dashboard_jobs, {jobs: jobs, only: only}
+          = render :dashboard_jobs, {current: current, queue: queue, only: only}
 
       .row
         #box-tokens.col-md-6

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -1,9 +1,10 @@
-- count_all = $queue.all_size
+- count_all = $queue.jobs_count
 - counts_by_status = $queue.counts_by_status
+-# jobs_without_queue = jobs
 
 
 %h2
-  Jobs  
+  Jobs table  
 
   .btn-group.btn-group-md
     - klass = only.empty? ? "btn-info" : ""
@@ -16,6 +17,7 @@
         #{status} (#{count})
 
 %table.table.table-striped.table-hover.table-condensed#jobs
+
   %thead
     %tr
       %th ID
@@ -23,13 +25,23 @@
       %th source
       %th <=>
       %th target
-      %th date
+      %th queued
       %th{width: 220} status
       %th{width: 150} error
       %th.text-right size
       %th.text-right bitrate
       %th info
 
-  - if true
-    %tbody.current
-      = render :dashboard_table, {jobs: jobs}
+
+    - unless queue.empty?
+      %tbody.jobs
+        = render :dashboard_table, {jobs: queue.reverse}
+
+        %thead
+          %tr
+            %td{colspan: 13}
+              %br
+
+    - unless current.empty?
+      %tbody.jobs
+        = render :dashboard_table, {jobs: current.reverse}