rest-ftp-daemon 0.85.2 → 0.90.1

data/lib/rest-ftp-daemon/job_queue.rb

@@ -1,7 +1,10 @@
 require 'thread'
+require 'securerandom'
 
 module RestFtpDaemon
   class JobQueue < Queue
+    attr_reader :queued
+    attr_reader :popped
 
     def initialize
       # # Logger
@@ -17,6 +20,12 @@ module RestFtpDaemon
       self.taint
       @mutex = Mutex.new
 
+      # Identifiers generator
+      @last_id = 0
+      #@prefix = SecureRandom.hex(IDENT_JOB_LEN)
+      @prefix = Helpers.identifier IDENT_JOB_LEN
+      info "queue initialized with prefix: #{@prefix}"
+
       # Mutex for counters
       @counters = {}
       @mutex_counters = Mutex.new
@@ -39,6 +48,13 @@ module RestFtpDaemon
 
     end
 
+    def generate_id
+      rand(36**8).to_s(36)
+      @last_id ||= 0
+      @last_id += 1
+      prefixed_id @last_id
+    end
+
     def counter_add name, value
       @mutex_counters.synchronize do
         @counters[name] ||= 0
@@ -60,27 +76,45 @@ module RestFtpDaemon
       end
     end
 
-    def by_status status
+    # def by_status status
+    #   return [] if status.nil?
+
+    #   # Select jobs from the queue if their status is (status)
+    #   all.select { |item| item.get(:status) == status.to_sym }
+    # end
+
+    def popped_reverse_sorted_by_status status
       return [] if status.nil?
 
       # Select jobs from the queue if their status is (status)
-      all.select { |item| item.get(:status) == status.to_sym }
+      ordered_popped.reverse.select { |item| item.get(:status) == status.to_sym }
     end
 
-    def queued
-      @queued
-    end
-    def popped
-      @popped
+    def popped_counts_by_status
+      statuses = {}
+      @popped.group_by { |job| job.get(:status) }.map { |status, jobs| statuses[status] = jobs.size }
+      statuses
     end
+
     def all
+      # queued2 = @queued.clone
+      # return queued2.merge(@popped)
       @queued + @popped
     end
     def all_size
       @queued.length + @popped.length
     end
 
-    def push(obj)
+    def find_by_id id, prefixed = false
+      # Build a prefixed id if expected
+      id = prefixed_id(id) if prefixed
+      info "find_by_id (#{id}, #{prefixed}) > #{id}"
+
+      # Search in both queues
+      @queued.select { |item| item.id == id }.last || @popped.select { |item| item.id == id }.last
+    end
+
+    def push obj
       # Check that item responds to "priorty" method
       raise "JobQueue.push: object should respond to priority method" unless obj.respond_to? :priority
 
@@ -97,11 +131,7 @@ module RestFtpDaemon
     alias << push
     alias enq push
 
-    #
-    # Retrieves data from the queue.  If the queue is empty, the calling thread is
-    # suspended until data is pushed onto the queue.  If +non_block+ is true, the
-    # thread isn't suspended, and an exception is raised.
-    #
+
     def pop(non_block=false)
       @mutex.synchronize do
         while true
@@ -110,7 +140,7 @@ module RestFtpDaemon
             @waiting.push Thread.current
             @mutex.sleep
           else
-            return pick
+            return pick_one
           end
         end
       end
@@ -130,9 +160,20 @@ module RestFtpDaemon
       @waiting.size
     end
 
+    def ordered_queue
+      @queued.sort_by { |item| [item.priority.to_i, - item.id.to_i] }
+    end
+
+    def ordered_popped
+      @popped.sort_by { |item| [item.get(:updated_at)] }
+    end
 
   protected
 
+    def prefixed_id id
+      "#{@prefix}.#{id}"
+    end
+
     def conchita_loop
       info "conchita starting with: #{@conchita.inspect}"
       loop do
@@ -175,21 +216,20 @@ module RestFtpDaemon
       @logger.add(Logger::INFO, "#{'  '*(level+1)} #{message}", progname) unless @logger.nil?
     end
 
-    def pick  # called inside a mutex/sync
+    def pick_one  # called inside a mutex/sync
       # Sort jobs by priority and get the biggest one
-      picked = @queued.sort { |a,b| a.priority.to_i <=> b.priority.to_i }.last
+      picked = ordered_queue.last
       return nil if picked.nil?
 
-      # Delete it from the queue
+      # Move it away from the queue to the @popped array
       @queued.delete_if { |item| item == picked }
-
-      # Stack it to popped items
       @popped.push picked
 
       # Return picked
       picked
     end
 
+
   private
 
     def info message, level = 0

data/lib/rest-ftp-daemon/notification.rb

@@ -32,12 +32,13 @@ module RestFtpDaemon
 
       # Params
       body = {
-        id:       params[:id],
+        id:       params[:id].to_s,
         signal:   params[:signal],
         error:    params[:error],
-        host:     get_hostname,
+        host:     Settings['host'].to_s,
         }
-      body[:status] = params[:status] unless params[:status].empty? || params[:status].nil?
+      body[:status] = params[:status] if (params[:status].is_a? Enumerable)
+      # && (!params[:status].empty?)
 
       # Send message in a thread
       Thread.new do |thread|
@@ -59,9 +60,5 @@ module RestFtpDaemon
 
   protected
 
-    def get_hostname
-      `hostname`.chomp
-    end
-
   end
 end

data/lib/rest-ftp-daemon/static/css/bootstrap.css

@@ -280,8 +280,8 @@ html {
 }
 body {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 12px;
-  line-height: 1.42857143;
+  font-size: 11px;
+  line-height: 1.3;
   color: #333333;
   background-color: #ffffff;
 }

data/lib/rest-ftp-daemon/static/css/main.css

@@ -0,0 +1,17 @@
+.text-right {
+  text-align: right;
+  }
+
+.transfer-method {
+  display: inline-block;
+  width: 35px;
+  padding: 4px 1px;
+  font-weight: normal;
+  }
+
+.nobr {
+  white-space: nowrap;
+  }
+.fixed {
+  font-family:monospace;
+  }

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -3,8 +3,9 @@
   %head
     %meta{:charset => "utf-8"}/
     %link{ href:"/css/bootstrap.css" , rel: "stylesheet"}
+    %link{ href:"/css/main.css" , rel: "stylesheet"}
 
-    %title="#{Settings.name} dashboard"
+    %title="#{Settings['host']} [#{Settings.namespace}] #{APP_NAME}"
 
     :css
 
@@ -47,6 +48,10 @@
         .btn.btn-default.btn-success IP
         .btn.btn-default= @info_ipaddr
 
+      .btn-group.btn-group-sm
+        .btn.btn-default.btn-success Host
+        .btn.btn-default= Settings['host']
+
       .btn-group.btn-group-sm
         .btn.btn-default.btn-warning Load
         .btn.btn-default= @info_load.round(1)
@@ -65,9 +70,43 @@
 
 
       .row
-        .col-md-9
-          = render :dashboard_jobs, {jobs: @jobs, counts: @counts}
-        .col-md-3
+        .col-md-10
+
+          %h2
+            Jobs  
+
+            .btn-group.btn-group-md
+              - klass = @only.nil? ? "btn-info" : ""
+              %a.btn.btn-default{href: "?only=", class: klass}
+                ALL (#{@jobs_queued.size} + #{@jobs_popped.size})
+            .btn-group.btn-group-md
+              - @counts.each do |status, count|
+                - klass = (status == @only) ? "btn-info" : ""
+                %a.btn.btn-default{href: "?only=#{status}", class: klass}
+                  #{status} (#{count})
+
+          %table.table.table-striped.table-hover.table-condensed
+            %tr
+              %th JID
+              %th prio
+              %th worker
+              %th source
+              %th target
+              %th status
+              %th error
+              %th progress
+              %th.text-right size
+              %th.text-right bitrate
+
+            - if @only.nil? && !@jobs_queued.empty?
+              = render :dashboard_jobs, {jobs: @jobs_queued, counts: @counts}
+              %tr
+                %th{colspan: 10}
+
+            = render :dashboard_jobs, {jobs: @jobs_popped, counts: @counts}
+
+
+        .col-md-2
           = render :dashboard_workers, {}
 
       = render :dashboard_tokens, {tokens: Settings.endpoints || {}}

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -1,75 +1,56 @@
-%h2
-  Jobs
-  = " "
-
-  .btn-group.btn-group-md
-    - klass = @only.nil? ? "btn-info" : ""
-    %a.btn.btn-default{href: "?only=", class: klass}
-      ALL (#{jobs.size})
-
-      =# jobs.size
-
-  .btn-group.btn-group-md
-    - counts.each do |status, count|
-      - klass = (status == @only) ? "btn-info" : ""
-      %a.btn.btn-default{href: "?only=#{status}", class: klass}
-        #{status} (#{count})
-
-%table.table.table-striped.table-hover.table-condensed
-
-  %tr
-    %th= "ID"
-    %th= "priority"
-    %th= "source"
-    %th= "target"
-    %th= "mode"
-    %th= "size"
-    %th= "status"
-    %th= "error"
-    %th= "progress"
-    %th= "worker"
-    %th
-
-  - jobs.each do |job|
-    - error = job.get :error
-    - status = job.get :status
-    - size = job.get :transfer_size
-    - progress = job.get :progress
-
-    - if error!=0 && !error.nil?
-      - trclass = "danger"
-    - elsif status == :uploading
-      - trclass = "info"
-    - elsif status == :finished
-      - trclass = "success"
-    - else
-      - trclass = "warning"
-
-    %tr{class: trclass}
-
-      %td= job.id
-      %td= job.get :priority
-      %td{title: job.get(:source_path)}
-        = job.get :source
-      %td{title: job.get(:target_url)}
-        = job.get :target
-      %td
-        .label.label-info= job.get :source_method
-        .label.label-primary= job.get :target_method
-      %td= Helpers.format_bytes(size, "B")
-      %td= status
-      %td= error
-      %td
-        - unless progress.nil?
-          .progress
-            .progress-bar{style:"width: #{progress}%;"}
-              = Helpers.format_bytes job.get(:transfer_sent), "B"
-      %td
-        - unless progress.nil?
-          = "#{progress}%"
-        - if (bitrate = job.get :transfer_bitrate)
-          = "|"
-          =  Helpers.format_bytes(bitrate, "bps")
-
-      %td= job.wid
+- jobs.each do |job|
+  - error = job.get :error
+  - status = job.get :status
+  - size = job.get :transfer_total
+  - progress = job.get :progress
+  - transfer_source_count = job.get(:transfer_source_count) || 0
+  - transfer_source_done = job.get(:transfer_source_done) || 0
+
+  - if error!=0 && !error.nil?
+    - trclass = "danger"
+  - elsif status == :uploading
+    - trclass = "info"
+  - elsif status == :finished
+    - trclass = "success"
+  - else
+    - trclass = "warning"
+
+  %tr{class: trclass}
+
+    %td= job.id
+
+    %td= job.get :priority
+
+    %td= job.wid
+
+    %td.fixed{title: job.get(:source_path)}
+      = Helpers.job_method_label job.get(:source_method)
+      = job.get :source
+
+    %td.fixed{title: job.get(:target_url)}
+      = Helpers.job_method_label job.get(:target_method)
+      = job.get :target
+
+    %td
+      = status
+      - if (transfer_source_count > 0) #&& (transfer_source_done < transfer_source_count)
+        %small= " #{transfer_source_done}/#{transfer_source_count}"
+
+    %td
+      = error
+      - unless progress.nil? || status == :finished
+        %small= "#{progress}%"
+
+    %td
+      - unless progress.nil?
+        .progress
+          .progress-bar{style:"width: #{progress}%;"}
+            = Helpers.format_bytes job.get(:transfer_sent), "B"
+
+    %td.nobr.text-right
+      = Helpers.format_bytes(size, "B")
+
+    %td.nobr.text-right
+      - if (bitrate = job.get :transfer_bitrate)
+        =  Helpers.format_bytes(bitrate, "bps")
 

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -12,7 +12,7 @@
 %table.table.table-striped.table-hover.table-condensed
 
   %tr
-    %th= "ID"
+    %th= "WID"
     %th= "status"
     %th= "job"
     %th= "time"

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -67,7 +67,7 @@ module RestFtpDaemon
           ex.backtrace.each do |line|
             info line, 1
           end
-          sleep 2
+          sleep 1
         else
 
         # Clean job status

data/rest-ftp-daemon.yml.sample

@@ -1,11 +1,11 @@
 defaults: &defaults
-  daemonize: 1
-  port: 3200
+  daemonize: true
+  port: 3000
   workers: 2
   #adminpwd: "admin"
-  pidfile: /tmp/rftpd.pid
   user: rftpd
   group: rftpd
+  host: <%= `hostname`.chomp.split('.').first %>
 
   transfer:
     #update_every_kb: 500
@@ -26,3 +26,4 @@ preprod:
 
 production:
   <<: *defaults
+  port: 3200