rest-ftp-daemon 0.300.3 → 0.302.0

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -9,9 +9,6 @@ module RestFtpDaemon
       # Call super
       super
 
-      # Use debug ?
-      @debug = (Conf.at :debug, :sftp) == true
-
       # Announce object
       log_debug "RemoteSFTP.initialize"
     end
@@ -22,7 +19,7 @@ module RestFtpDaemon
       log_debug "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
 
       # Debug level
-      verbosity = @debug ? Logger::INFO : false
+      verbosity =  @debug ? Logger::INFO : false
 
       # Connect remote server
       @sftp = Net::SFTP.start(@url.host.to_s, @url.user.to_s,

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -59,14 +59,14 @@ module RestFtpDaemon
 
     def create_threads
       # Read configuration or initialize with empty hash
-      pools = Conf[:pools]
+      pools = Conf.at[:pools]
       pools = {} unless pools.is_a? Hash
 
       # Minimum one worker on DEFAULT_POOL
       if !(pools.is_a? Hash)
         log_error "create_threads: one JobWorker is the minimum (#{pools.inspect}"
       end
-      log_info "WorkerPool creating workers - JobWorker #{pools.to_hash.inspect}"
+      log_info "WorkerPool creating all workers with #{pools.to_hash.inspect}"
 
       # Start ConchitaWorker and ReporterWorker
       create_thread :conchita, ConchitaWorker
@@ -77,7 +77,7 @@ module RestFtpDaemon
       pools.each do |pool, count|
         count.times do
           wid = next_worker_id
-          @workers[wid] = create_thread wid, JobWorker, pool
+          create_thread(wid, TransferWorker, pool)
         end
       end
 
@@ -85,23 +85,13 @@ module RestFtpDaemon
       log_error "EXCEPTION: #{ex.message}", ex.backtrace
     end
 
-    # def create_worker_thread wid, pool
-    #   Thread.new wid do
-    #     begin
-    #       worker = JobWorker.new wid, pool
-    #       #log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
-    #     rescue StandardError => ex
-    #       log_error "JobWorker EXCEPTION: #{ex.message} #{e.backtrace}"
-    #     end
-    #   end
-    # end
-
     def create_thread wid, klass, pool = nil
       # Spawn thread and add it to my index
-      log_info "spawning #{klass.name} [#{wid}] [#{pool}]"
+      log_info "spawning #{klass.name} wid[#{wid}] pool[#{pool}]"
       @workers[wid] = Thread.new do
         begin
           worker = klass.new wid, pool
+          sleep 0.1
         rescue StandardError => ex
           log_error "#{klass.name} EXCEPTION: #{ex.message}"
         end

data/lib/rest-ftp-daemon/{worker_conchita.rb → workers/conchita.rb}

@@ -1,35 +1,33 @@
 module RestFtpDaemon
 
   # Worker used to clean up the queue deleting expired jobs
-  class ConchitaWorker < Worker
+  class ConchitaWorker < Shared::WorkerBase
 
-    def initialize wid, pool = nil
-      # Call dady and load my conf
-      super
+  protected
 
-      # Start main loop
-      log_info "#{self.class.name} starting", @config
-      start
-    end
+    def worker_init
+      # Load corker conf
+      config_section :conchita
 
-  protected
+      # Check that everything is OK
+      return "invalid timer" unless @config[:timer].to_i > 0
+      return false
+    end
 
-    # def log_prefix
-    #  [
-    #   Thread.current.thread_variable_get(:wid),
-    #   nil,
-    #   nil
-    #   ]
-    # end
+    def worker_after
+      # Sleep for a few seconds
+      worker_status WORKER_STATUS_WAITING
+      sleep @config[:timer]
+    end
 
-    def work
+    def worker_process
       # Announce we are working
       worker_status WORKER_STATUS_CLEANING
 
       # Cleanup queues according to configured max-age
-      $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED),  @debug
-      $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED),    @debug
-      $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED),    @debug
+      $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED),  @config[:debug]
+      $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED),    @config[:debug]
+      $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED),    @config[:debug]
 
       # Force garbage collector
       GC.start if @config["garbage_collector"]
@@ -37,10 +35,10 @@ module RestFtpDaemon
     rescue StandardError => e
       log_error "EXCEPTION: #{e.inspect}"
       sleep 1
-    else
-      wait_according_to_config
     end
 
+  private
+
     def maxage status
       @config["clean_#{status}"] || 0
     end

data/lib/rest-ftp-daemon/{worker_reporter.rb → workers/reporter.rb}

@@ -1,28 +1,26 @@
 module RestFtpDaemon
 
-  # Worker used to clean up the queue deleting expired jobs
-  class ReporterWorker < Worker
+  # Worker used to report metrics to various services
+  class ReporterWorker < Shared::WorkerBase
 
-    def initialize wid, pool = nil
-      # Call dady and load my conf
-      super
+  protected
 
-      # Start main loop
-      log_info "#{self.class.name} starting", @config
-      start
-    end
+    def worker_init
+      # Load corker conf
+      config_section :reporter
 
-  protected
+      # Check that everything is OK
+      return "invalid timer" unless @config[:timer].to_i > 0
+      return false
+    end
 
-    # def log_prefix
-    #  [
-    #   Thread.current.thread_variable_get(:wid),
-    #   nil,
-    #   nil
-    #   ]
-    # end
+    def worker_after
+      # Sleep for a few seconds
+      worker_status WORKER_STATUS_WAITING
+      sleep @config[:timer]
+    end
 
-    def work
+    def worker_process
       # Announce we are working
       worker_status WORKER_STATUS_REPORTING
 
@@ -32,23 +30,14 @@ module RestFtpDaemon
     rescue StandardError => e
       log_error "EXCEPTION: #{e.inspect}"
       sleep 1
-    else
-      wait_according_to_config
-    end
-
-    def maxage status
-      @config["clean_#{status}"] || 0
     end
 
   private
 
     def do_metrics
-      # Get common metrics
-      log_info "collecting metrics"
+      # Get common metrics and dump them to logs
       metrics = Metrics.sample
-
-      # Dump metrics to logs
-      log_debug "collected metrics", metrics
+      log_info "collected metrics", metrics
 
       # Transpose metrics to NewRelic metrics
       report_newrelic(metrics) if Conf.newrelic_enabled?
@@ -63,7 +52,11 @@ module RestFtpDaemon
           metrics_newrelic[name] = value
         end
       end
-      log_debug "reported [#{metrics.size}] metrics to NewRelic", metrics_newrelic
+      if @config[:debug]
+        log_debug "reported following metrics to NewRelic", metrics_newrelic
+      else
+        log_debug "reported all metrics to NewRelic"
+      end
     end
 
   end

data/lib/rest-ftp-daemon/{worker_job.rb → workers/transfer.rb}

@@ -1,42 +1,35 @@
 module RestFtpDaemon
 
   # Worker used to process Jobs
-  class JobWorker < Worker
+  class TransferWorker < Shared::WorkerBase
 
-    def initialize wid, pool
-      # Call dady and load my conf
-      super
-
-      # Timeout and retry config
-      @timeout    = (Conf.at(:transfer, :timeout) rescue nil)
-      @retry      = (Conf.at(:retry) rescue {})
+  protected
 
-      # Retry config
-      @retry_on_errors   = Conf.at(:retry, :on_errors)
-      @retry_max_age     = Conf.at(:retry, :max_age)
-      @retry_max_runs    = Conf.at(:retry, :max_runs)
-      @retry_delay       = Conf.at(:retry, :delay)
+    def worker_init
+      # Load standard config
+      config_section    :transfer
+      @endpoints        = Conf[:endpoints]
 
+      # Timeout and retry config
+      return "invalid timeout" unless @config[:timeout].to_i > 0
 
-      # Start main loop
-      log_info "JobWorker initializing", {
-        wid: wid,
-        pool: pool,
-        timeout: @timeout
+      # Log that
+      log_info "JobWorker worker_init", {
+        pool: @pool,
+        timeout: @config[:timeout]
       }
-      start
-    end
-
-  protected
 
+      return false
+    end
 
-    def load_config wid
-      # Do nothing
+    def worker_after
+      # Clean worker status
+      worker_jid nil
     end
 
   private
 
-    def work
+    def worker_process
       # Wait for a job to be available in the queue
       worker_status WORKER_STATUS_WAITING
       job = $queue.pop @pool
@@ -52,18 +45,18 @@ module RestFtpDaemon
       if !job.error
         #log_info "job succeeded"
 
-      elsif !(@retry_on_errors.is_a?(Enumerable) && @retry_on_errors.include?(job.error))
+      elsif !(@config[:retry_on].is_a?(Enumerable) && @config[:retry_on].include?(job.error))
         log_error "not retrying: error not eligible"
 
-      elsif @retry_max_age && (job.age >= @retry_max_age)
-        log_error "not retrying: max_age reached (#{@retry_max_age} s)"
+      elsif @config[:retry_for] && (job.age >= @config[:retry_for])
+        log_error "not retrying: max_age reached (#{@config[:retry_for]} s)"
 
-      elsif @retry_max_runs && (job.runs >= @retry_max_runs)
-        log_error "not retrying: max_runs reached (#{@retry_max_runs} tries)"
+      elsif @config[:retry_max] && (job.runs >= @config[:retry_max])
+        log_error "not retrying: max_runs reached (#{@config[:retry_max]} tries)"
 
       else
         # Delay cannot be negative, and will be 1s minimum
-        retry_after = [@retry_delay || DEFAULT_RETRY_DELAY, 1].max
+        retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
         log_info "retrying job: waiting for #{retry_after} seconds"
 
         # Wait !
@@ -74,9 +67,6 @@ module RestFtpDaemon
         $queue.requeue job
       end
 
-      # Clean worker status
-      worker_jid nil
-
     rescue StandardError => ex
       log_error "WORKER UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
       worker_status WORKER_STATUS_CRASHED
@@ -89,7 +79,7 @@ module RestFtpDaemon
       job.wid = Thread.current.thread_variable_get :wid
 
       # Processs this job protected by a timeout
-      Timeout.timeout(@timeout, RestFtpDaemon::JobTimeout) do
+      Timeout.timeout(@config[:timeout], RestFtpDaemon::JobTimeout) do
         job.process
       end
 

data/lib/shared/conf.rb

@@ -10,6 +10,7 @@ module Shared
 
   class Conf
     extend Chamber
+    PIDFILE_DIR = "/tmp/"
 
     class << self
       attr_accessor :app_env
@@ -21,7 +22,6 @@ module Shared
       attr_reader   :app_spec
       attr_reader   :files
       attr_reader   :host
-
     end
 
     def self.init app_root
@@ -51,11 +51,12 @@ module Shared
       fail ConfigMissingParameter, "gemspec: missing version" unless @app_ver
 
       # Now we know app_name, initalize app_libs
-      @app_libs = File.expand_path( @app_root + "/lib/#{@app_name}/" )
+      @app_libs = File.expand_path("lib/#{@app_name}/", @app_root)
 
       # Add other config files
-      add_default_config
-      add_etc_config
+      #add_default_config
+      add_config generate(:config_defaults)
+      add_config generate(:config_etc)
 
       # Return something
       return @app_name
@@ -64,17 +65,20 @@ module Shared
     def self.prepare args = {}
       ensure_init
 
-      # Add extra config file
-      add_extra_config args[:config]
-
-      # Load configuration files
-      load_files
+      # Add extra config file and load them all
+      add_config args[:config]
+      reload!
 
       # Set Rack env
       ENV["RACK_ENV"] = @app_env.to_s
 
+      # Set up encodings
+      Encoding.default_internal = "utf-8"
+      Encoding.default_external = "utf-8"
+
       # Init New Relic
-      prepare_newrelic self[:newrelic], self.at(:logs, :newrelic)
+      newrelic_logfile = File.expand_path(Conf[:logs][:newrelic], Conf[:logs][:path])
+      prepare_newrelic self[:newrelic], newrelic_logfile
 
       # Try to access any key to force parsing of the files
       self[:dummy]
@@ -104,53 +108,55 @@ module Shared
 
     def self.newrelic_enabled?
       ensure_init
-      !!self[:newrelic]
+      self[:newrelic] && self[:newrelic][:licence]
     end
 
     # Defaults generators
-    def self.gen_pidfile
-      ensure_init
-      "/tmp/#{@app_name}-#{@host}-#{self[:port]}.pid"
-    end
-    def self.gen_config_etc
+    def self.generate what
       ensure_init
-      "/etc/#{@app_name}.yml"
-    end
-    def self.gen_config_sample
-      ensure_init
-      "#{@app_root}/#{@app_name}.sample.yml"
-    end
-    def self.gen_config_message
-      config_etc = gen_config_etc
-      config_sample = gen_config_sample
-      return "
-A default configuration is available here: #{config_sample}.
-You should copy it to the default location: #{config_etc}.
-sudo cp #{config_sample} #{config_etc}
-"
+      return case what
+
+      when :user_agent
+        "#{@app_name}/#{@app_ver}" if @app_name && @app_ver
+
+      when :config_defaults
+        "#{@app_root}/defaults.yml" if @app_root
+
+      when :config_etc
+        "/etc/#{@app_name}.yml" if @app_name
+
+      when :process_name
+        parts = [@app_name, @app_env]
+        parts << self[:port] if self[:port]
+        parts.join('-')
+
+      when :pidfile
+        process_name = self.generate(:process_name)
+        File.expand_path "#{process_name}.pid", PIDFILE_DIR
+
+      when :config_message
+        config_defaults = self.generate(:config_defaults)
+        config_etc = self.generate(:config_etc)
+
+        "A default configuration is available (#{config_defaults}) and can be copied to the default location (#{config_etc}): \n sudo cp #{config_defaults} #{config_etc}"
+
+      end
     end
 
+
   protected
 
     def self.load_files
       load files: @files, namespaces: { environment: @app_env }
     end
 
-    def self.add_default_config
-      @files << "#{@app_root}/defaults.yml" if @app_root
-    end
-
-    def self.add_etc_config
-      @files << File.expand_path("/etc/#{@app_name}.yml") if @app_name
-    end
-
-    def self.add_extra_config path
-      @files << File.expand_path(path) if path
+    def self.add_config path
+      @files << File.expand_path(path) if path && File.readable?(path)
     end
 
     def self.prepare_newrelic section, logfile
       # Disable NewRelic if no config present
-      unless section.is_a?(Hash) && section[:licence]
+      unless self.newrelic_enabled?
         ENV["NEWRELIC_AGENT_ENABLED"] = "false"
         return
       end