rest-ftp-daemon 0.213.0 → 0.214.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8cdf3cbfd6993183cfe6410fc11ce2743f0d3ca2
-  data.tar.gz: 07938cd1ed765a2939e8bdc2589069e9379dcd7b
+  metadata.gz: a453a7be5c87798246b4603a6787365b5829f58f
+  data.tar.gz: 3583043f7aa342bc2992fc5a8b0599a19acfe693
 SHA512:
-  metadata.gz: 22536522c3a2cc04c73876305ce7f9217f9f8a12ec141c0d8ea8113c7908c94aa7569477051d179316afffb2efd2c836fb940418aa2d00acd74485b95e45238a
-  data.tar.gz: 4835fa78f98b4ab22b57be1456e438c65687756ec919435e5dd6cbf5880c7795724ee336fb334ca71df0667b9a8aa681e4a199b12b45bdfc5f696f81378c96f2
+  metadata.gz: 94e64e9196678fa87b03d0ada12c1f7b4064e5c88d1996731b060a9703c901d858e2f0aab525f0b3ad5a12e15aad076e51d9b6e9fcd2292cad24788c78606368
+  data.tar.gz: 3933f23e1bbe6c039ef4f94c69e0bf682504514e8718a9df14d9c9c4f7466357dbf2a190170a8bc143975b5c1ff6543b52c15d4627f86f4afb360f11c87dd3ad

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.212.0)
+    rest-ftp-daemon (0.214.0)
       double-bag-ftps
       facter
       get_process_mem

data/config.ru

@@ -3,8 +3,10 @@ load_path_libs = File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
 $LOAD_PATH.unshift(load_path_libs) unless $LOAD_PATH.include?(load_path_libs)
 require 'rest-ftp-daemon'
 
-# Create queue and worker pool
+# Create global queue
 $queue = RestFtpDaemon::JobQueue.new
+
+# Initialize workers and conchita subsystem
 $pool = RestFtpDaemon::WorkerPool.new(Settings.workers || APP_WORKERS)
 
 # Rack reloader

data/lib/rest-ftp-daemon.rb

@@ -14,10 +14,13 @@ require 'syslog'
 require 'net/ftp'
 require 'net/http'
 require 'double_bag_ftps'
+require 'thread'
+require 'securerandom'
 
 require 'newrelic_rpm'
 require 'get_process_mem'
 
+
 # Project's libs
 require 'rest-ftp-daemon/constants'
 require 'rest-ftp-daemon/settings'
@@ -27,6 +30,7 @@ require 'rest-ftp-daemon/paginate'
 require 'rest-ftp-daemon/uri'
 require 'rest-ftp-daemon/job_queue'
 # require 'rest-ftp-daemon/worker'
+require 'rest-ftp-daemon/conchita'
 require 'rest-ftp-daemon/worker_pool'
 require 'rest-ftp-daemon/logger'
 require 'rest-ftp-daemon/logger_pool'

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -22,17 +22,17 @@ module RestFtpDaemon
         @only = params["only"].to_s
 
         # Get jobs for this view, order jobs by their weights
-        result = $queue.filter_jobs @only
+        result = $queue.filter_jobs(@only).reverse
 
         # Provide queue only if no filtering set
         @queue = []
-        @queue = $queue.queue if @only.empty?
+        @queue = $queue.queue.reverse if @only.empty?
 
         # Get workers status
         @worker_variables = $pool.worker_variables
 
         # Build paginator
-        @paginate = Paginate.new result.reverse
+        @paginate = Paginate.new result
         @paginate.only = params["only"]
         @paginate.page = params["page"]
 

data/lib/rest-ftp-daemon/conchita.rb

@@ -0,0 +1,65 @@
+module RestFtpDaemon
+  class Conchita
+
+    def initialize
+      # Logger
+      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
+
+      # Conchita configuration
+      @conchita = Settings.conchita
+      if @conchita.nil?
+        return info "conchita: missing conchita.* configuration"
+      elsif @conchita[:timer].nil?
+        return info "conchita: missing conchita.timer value"
+      end
+
+      # Start main loop
+      info "initialized #{@conchita.inspect}"
+      cleanup
+    end
+
+  protected
+
+    def maxage status
+      @conchita["clean_#{status.to_s}"] || 0
+    end
+
+    def cleanup
+      loop do
+        #info "cleanup"
+        # info "conchita_loop: cleanup "
+        $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED)
+        $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED)
+        $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED)
+
+        # Force garbage collector
+        GC.start if @conchita["garbage_collector"]
+
+        # Sleep for a few seconds
+        sleep @conchita[:timer]
+      end
+    end
+
+    # def conchita_gc
+    #   # Read config state
+    #   proceed = @conchita["clean_garbage"] || false
+    #   #info "conchita_clean status[#{status.to_s}] \t maxage[#{maxage}] s"
+    #   return unless proceed
+
+    #   # Trig Ruby's garbage collector
+    #   info "conchita_gc forced garbage collecting"
+    #   GC.start
+    # end
+
+    def info message, lines = []
+      return if @logger.nil?
+
+      # Forward to logger
+      @logger.info_with_id message,
+        wid: :conchita,
+        lines: lines,
+        origin: self.class.to_s
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.213.0"
+APP_VER = "0.214.0"
 
 
 # Jobs and workers
@@ -17,10 +17,11 @@ JOB_STATUS_QUEUED       = :queued
 
 # Logging and startup
 LOG_PIPE_LEN            = 10
-LOG_COL_WID             = 4
+LOG_COL_WID             = 8
 LOG_COL_JID             = JOB_IDENT_LEN+3+2
 LOG_COL_ID              = 6
 LOG_TRIM_LINE           = 80
+LOG_DUMPS               = File.dirname(__FILE__) + '/../../log/'
 
 
 # Notifications
@@ -39,13 +40,14 @@ JOB_STYLES = {
   }
 WORKER_STYLES = {
   :waiting              => :success,
-  :processing           => :info,
+  :working              => :info,
   :crashed              => :danger,
   :done                 => :success,
   :dead                 => :danger
   }
 PAGINATE_MAX            = 30
 
+
 # Configuration defaults
 DEFAULT_WORKER_TIMEOUT  = 3600
 DEFAULT_FTP_CHUNK       = 2048

data/lib/rest-ftp-daemon/job.rb

@@ -440,9 +440,10 @@ module RestFtpDaemon
     end
 
     def ftp_finished
-      # Close FTP connexion
+      # Close FTP connexion and free up memory
       @ftp.close
       info "Job.ftp_finished closed"
+      @ftp = nil
 
       # FTP debug mode ?
       if @ftp_debug_enabled

data/lib/rest-ftp-daemon/job_queue.rb

@@ -1,10 +1,5 @@
-require 'thread'
-require 'securerandom'
-
 module RestFtpDaemon
   class JobQueue < Queue
-    # attr_reader :queued
-    # attr_reader :popped
 
     attr_reader :queue
     attr_reader :jobs
@@ -34,23 +29,6 @@ module RestFtpDaemon
       # Mutex for counters
       @counters = {}
       @mutex_counters = Mutex.new
-
-      # Conchita configuration
-      @conchita = Settings.conchita
-      if @conchita.nil?
-        info "conchita: missing conchita.* configuration"
-      elsif @conchita[:timer].nil?
-        info "conchita: missing conchita.timer value"
-      else
-        Thread.new {
-          begin
-            conchita_loop
-          rescue Exception => e
-            info "CONCHITA EXCEPTION: #{e.inspect}"
-          end
-          }
-      end
-
     end
 
     def generate_id
@@ -187,69 +165,46 @@ module RestFtpDaemon
       @waiting.size
     end
 
-  protected
-
-    def prefixed_id id
-      "#{@prefix}.#{id}"
-    end
-
-    def sort_queue!
-      @mutex_counters.synchronize do
-        @queue.sort_by! &:weight
-      end
-    end
-
-    def conchita_loop
-      info "conchita starting with: #{@conchita.inspect}"
-      loop do
-        # Do the cleanup locking the queues
-        # info "conchita: cleanup expired jobs"
-        @mutex.synchronize do
-          conchita_clean JOB_STATUS_FINISHED
-          conchita_clean JOB_STATUS_FAILED
-          conchita_clean JOB_STATUS_QUEUED
-        end
-        sleep @conchita[:timer]
-      end
-    end
-
-    def conchita_clean status     # FIXME: clean both @jobs and @queue
+    def expire status, maxage
+# FIXME: clean both @jobs and @queue
       # Init
-      return if status.nil?
-
-      # Read config state
-      maxage = @conchita["clean_#{status.to_s}"] || 0
-      #info "conchita_clean status[#{status.to_s}] \t maxage[#{maxage}] s"
-      return unless maxage > 0
+      return if status.nil? || maxage <= 0
 
-      # Delete jobs from the queue if their status is (status)
-      @jobs.delete_if do |job|
+      # Compute oldest possible birthday
+      before = Time.now - maxage.to_i
+      # info "conchita_clean: SELECT status[#{status.to_s}] before[#{before}]"
 
-        # Skip if wrong status
-        next unless job.status == status.to_sym
-
-        # Skip if updated_at invalid
-        next if job.updated_at.nil?
+      @mutex.synchronize do
+        # Delete jobs from the queue when they match status and age limits
+        @jobs.delete_if do |job|
 
-        # Skip if not aged enough yet
-        age = Time.now - job.updated_at
-        next if age < maxage
+          # Skip if wrong status, updated_at invalid, or too young
+          next unless job.status == status.to_sym
+          next if job.updated_at.nil?
+          next if job.updated_at > before
 
-        # Ok, we have to clean it up ..
-        info "conchita_clean status[#{status.to_s}] maxage[#{maxage}] job[#{job.id}] age[#{age}]"
+          # Ok, we have to clean it up ..
+          info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
+          info "       + unqueued" if @queue.delete(job)
 
-        # Remove it from the queue if present
-        job_in_queue = @queue.delete job
-        info "   removed queued job [#{job.id}]" unless job_in_queue.nil?
+          true
+        end
 
-        # Accept to delete it from @jobs
-        true
       end
 
     end
 
+  protected
+
+    def prefixed_id id
+      "#{@prefix}.#{id}"
+    end
 
-  private
+    def sort_queue!
+      @mutex_counters.synchronize do
+        @queue.sort_by! &:weight
+      end
+    end
 
     def info message, lines = []
       return if @logger.nil?
@@ -264,6 +219,8 @@ module RestFtpDaemon
     if Settings.newrelic_enabled?
       add_transaction_tracer :push, :category => :task
       add_transaction_tracer :pop, :category => :task
+      add_transaction_tracer :sort_queue!, :category => :task
+      add_transaction_tracer :expire, :category => :task
     end
 
   end

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -24,9 +24,14 @@ module RestFtpDaemon
 
       # And the formatter
       logger.formatter = proc do |severity, datetime, progname, message|
-        stamp = datetime.strftime("%Y-%m-%d %H:%M:%S")
-        field_pipe = "%-#{LOG_PIPE_LEN.to_i}s" % progname
-        "#{stamp}\t#{field_pipe}\t#{message}\n"
+        # stamp = datetime.strftime("%Y-%m-%d %H:%M:%S")
+        # field_pipe = "%-#{LOG_PIPE_LEN.to_i}s" % progname
+        # "#{stamp}\t#{field_pipe}\t#{message}\n"
+        "%s\t%-#{LOG_PIPE_LEN.to_i}s\t%s\n" % [
+          datetime.strftime("%Y-%m-%d %H:%M:%S"),
+          progname,
+          message,
+        ]
       end
 
       # Finally return this logger

data/lib/rest-ftp-daemon/static/css/main.css

@@ -14,8 +14,9 @@
   white-space: nowrap;
   }
 
-table tr td, .fixed {
-  font-family: 'Inconsolata', sans-serif;
+body, table tr td, .fixed {
+  /*font-family: 'Inconsolata', Verdana, sans-serif;*/
+  font-family: 'Verdana', Verdana, sans-serif;
   font-size: 1em;
   }
 
@@ -76,3 +77,9 @@ a.page.btn {
 .debug {
   border: 1px solid orange;
 }
+
+.table-condensed tbody > tr > td {
+  padding: 3px 3px;
+  }
+
+

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -42,7 +42,7 @@
 
         - unless @queue.empty?
           %tbody.jobs
-            = render :dashboard_table, {jobs: @queue.reverse}
+            = render :dashboard_table, {jobs: @queue}
 
             %thead
               %tr
@@ -51,4 +51,4 @@
 
         - unless jobs.empty?
           %tbody.jobs
-            = render :dashboard_table, {jobs: jobs.reverse}
+            = render :dashboard_table, {jobs: jobs}

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -17,14 +17,18 @@ module RestFtpDaemon
       # Prepare status hash and vars
       @statuses = {}
       @workers = {}
+      @conchita = nil
       @mutex = Mutex.new
       @counter = 0
       @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
 
       # Create worker threads
-      info "WorkerPool initializing with [#{number_threads}] workers and [#{@timeout}]s timeout"
+      info "WorkerPool creating worker threads [#{number_threads}] timeout [#{@timeout}]s"
       create_worker_threads number_threads
 
+      # Create conchita thread
+      info "WorkerPool creating conchita thread"
+      create_conchita_thread
     end
 
     def worker_variables
@@ -44,6 +48,7 @@ module RestFtpDaemon
   private
 
     def create_worker_threads n
+# FIXME counter instead of upto ?
       n.times do
         # Increment counter
         @mutex.synchronize do
@@ -77,6 +82,16 @@ module RestFtpDaemon
       end
     end
 
+    def create_conchita_thread
+      Thread.new do
+        begin
+          @conchita = Conchita.new
+        rescue Exception => e
+          info "CONCHITA EXCEPTION: #{e.inspect}"
+        end
+      end
+    end
+
     def work
       # Wait for a job to come into the queue
       worker_status :waiting

data/rest-ftp-daemon.yml.sample

@@ -20,7 +20,8 @@ defaults: &defaults
     # license: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
   conchita:
-    timer: 10
+    timer: 60
+    # garbage_collector: true
     # clean_failed: 3600
     # clean_finished: 3600
     # clean_queued: 86400

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.213.0
+  version: 0.214.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-30 00:00:00.000000000 Z
+date: 2015-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -231,6 +231,7 @@ files:
 - lib/rest-ftp-daemon/api/root.rb
 - lib/rest-ftp-daemon/api/routes.rb
 - lib/rest-ftp-daemon/api/status.rb
+- lib/rest-ftp-daemon/conchita.rb
 - lib/rest-ftp-daemon/constants.rb
 - lib/rest-ftp-daemon/exceptions.rb
 - lib/rest-ftp-daemon/helpers.rb