rest-ftp-daemon 0.250.5 → 0.300.1

data/lib/rest-ftp-daemon/worker_reporter.rb

@@ -0,0 +1,70 @@
+module RestFtpDaemon
+
+  # Worker used to clean up the queue deleting expired jobs
+  class ReporterWorker < Worker
+
+    def initialize wid, pool = nil
+      # Call dady and load my conf
+      super
+
+      # Start main loop
+      log_info "#{self.class.name} starting", @config
+      start
+    end
+
+  protected
+
+    # def log_prefix
+    #  [
+    #   Thread.current.thread_variable_get(:wid),
+    #   nil,
+    #   nil
+    #   ]
+    # end
+
+    def work
+      # Announce we are working
+      worker_status WORKER_STATUS_REPORTING
+
+      # Report metrics
+      do_metrics
+
+    rescue StandardError => e
+      log_error "EXCEPTION: #{e.inspect}"
+      sleep 1
+    else
+      wait_according_to_config
+    end
+
+    def maxage status
+      @config["clean_#{status}"] || 0
+    end
+
+  private
+
+    def do_metrics
+      # Get common metrics
+      log_info "collecting metrics"
+      metrics = Metrics.sample
+
+      # Dump metrics to logs
+      log_debug "collected metrics", metrics
+
+      # Transpose metrics to NewRelic metrics
+      report_newrelic(metrics) if Conf.newrelic_enabled?
+    end
+
+    def report_newrelic metrics
+      metrics_newrelic = {}
+      metrics.each do |group, pairs|
+        pairs.each do |key, value|
+          name = "rftpd/#{group}/#{key}"
+          ::NewRelic::Agent.record_metric(name, value)
+          metrics_newrelic[name] = value
+        end
+      end
+      log_debug "reported [#{metrics.size}] metrics to NewRelic", metrics_newrelic
+    end
+
+  end
+end

data/lib/shared/conf.rb

@@ -0,0 +1,195 @@
+# FIXME: files named with hyphens will not be found by Chamber for now
+require "chamber"
+
+module Shared
+  class ConfigMissingParameter    < StandardError; end
+  class ConfigOtherError          < StandardError; end
+  class ConfigParseError          < StandardError; end
+  class ConfigMultipleGemspec     < StandardError; end
+  class ConfigMissingGemspec      < StandardError; end
+
+  class Conf
+    extend Chamber
+
+    class << self
+      attr_accessor :app_env
+      attr_reader   :app_root
+      attr_reader   :app_libs
+      attr_reader   :app_name
+      attr_reader   :app_ver
+      attr_reader   :app_started
+      attr_reader   :app_spec
+      attr_reader   :files
+      attr_reader   :host
+
+    end
+
+    def self.init app_root
+      # Permanent flags
+      @initialized  = true
+      @app_started  = Time.now
+
+      # Default values
+      @files        ||= []
+      @app_name     ||= "app_name"
+      @app_env      ||= "production"
+      @host         ||= `hostname`.to_s.chomp.split(".").first
+
+      # Store and clean app_root
+      @app_root = File.expand_path(app_root)
+
+      # Try to find any gemspec file
+      matches   = Dir["#{@app_root}/*.gemspec"]
+      fail ConfigMissingGemspec, "gemspec file not found: #{gemspec_path}" if matches.size < 1
+      fail ConfigMultipleGemspec, "gemspec file not found: #{gemspec_path}" if matches.size > 1
+
+      # Load Gemspec (just the only match)
+      @spec     = Gem::Specification::load(matches.first)
+      @app_name = @spec.name
+      @app_ver  = @spec.version
+      fail ConfigMissingParameter, "gemspec: missing name" unless @app_name
+      fail ConfigMissingParameter, "gemspec: missing version" unless @app_ver
+
+      # Now we know app_name, initalize app_libs
+      @app_libs = File.expand_path( @app_root + "/lib/#{@app_name}/" )
+
+      # Add other config files
+      add_default_config
+      add_etc_config
+
+      # Return something
+      return @app_name
+    end
+
+    def self.prepare args = {}
+      ensure_init
+
+      # Add extra config file
+      add_extra_config args[:config]
+
+      # Load configuration files
+      load_files
+
+      # Set Rack env
+      ENV["RACK_ENV"] = @app_env.to_s
+
+      # Init New Relic
+      prepare_newrelic self[:newrelic], self.at(:logs, :newrelic)
+
+      # Try to access any key to force parsing of the files
+      self[:dummy]
+
+    rescue Psych::SyntaxError => e
+      fail ConfigParseError, e.message
+    rescue StandardError => e
+      fail ConfigOtherError, "#{e.message} \n #{e.backtrace.to_yaml}"
+    end
+
+    # Reload files
+    def self.reload!
+      ensure_init
+      load_files
+    end
+
+    def self.dump
+      ensure_init
+      to_hash.to_yaml(indent: 4, useheader: true, useversion: false )
+    end
+
+    # Direct access to any depth
+    def self.at *path
+      ensure_init
+      path.reduce(Conf) { |m, key| m && m[key.to_s] }
+    end
+
+    def self.newrelic_enabled?
+      ensure_init
+      !!self[:newrelic]
+    end
+
+    # Defaults generators
+    def self.gen_pidfile
+      ensure_init
+      "/tmp/#{@app_name}-#{@host}-#{self[:port]}.pid"
+    end
+    def self.gen_config_etc
+      ensure_init
+      "/etc/#{@app_name}.yml"
+    end
+    def self.gen_config_sample
+      ensure_init
+      "#{@app_root}/#{@app_name}.sample.yml"
+    end
+    def self.gen_config_message
+      config_etc = gen_config_etc
+      config_sample = gen_config_sample
+      return "
+A default configuration is available here: #{config_sample}.
+You should copy it to the default location: #{config_etc}.
+sudo cp #{config_sample} #{config_etc}
+"
+    end
+
+  protected
+
+    def self.load_files
+      load files: @files, namespaces: { environment: @app_env }
+    end
+
+    def self.add_default_config
+      @files << "#{@app_root}/defaults.yml" if @app_root
+    end
+
+    def self.add_etc_config
+      @files << File.expand_path("/etc/#{@app_name}.yml") if @app_name
+    end
+
+    def self.add_extra_config path
+      @files << File.expand_path(path) if path
+    end
+
+    def self.prepare_newrelic section, logfile
+      # Disable NewRelic if no config present
+      unless section.is_a?(Hash) && section[:licence]
+        ENV["NEWRELIC_AGENT_ENABLED"] = "false"
+        return
+      end
+
+      # Enable GC profiler
+      GC::Profiler.enable
+
+      # Enable module
+      ENV["NEWRELIC_AGENT_ENABLED"] = "true"
+      ENV["NEW_RELIC_MONITOR_MODE"] = "true"
+
+      # License
+      ENV["NEW_RELIC_LICENSE_KEY"] = section[:licence].to_s
+
+      # Build NewRelic app_name if not provided as-is
+      if section[:app_name]
+        ENV["NEW_RELIC_APP_NAME"] = section[:app_name].to_s
+      else
+        stack = []
+        stack << (section[:prefix] || @app_name)
+        stack << section[:platform] if section[:platform]
+        stack << @app_env
+        text = stack.join('-')
+        ENV["NEW_RELIC_APP_NAME"] = "#{text}-#{host};#{text}"
+      end
+
+      # Logfile
+      ENV["NEW_RELIC_LOG"] = logfile.to_s if logfile
+    end
+
+  private
+
+    def self.ensure_init
+      # Skip is already done
+      return if @initialized
+
+      # Go through init if not already done
+      self.init
+    end
+
+  end
+end

data/lib/shared/logger_formatter.rb

@@ -0,0 +1,31 @@
+module Shared
+  class LoggerFormatter
+
+    def self.call severity, datetime, progname, payload
+      # Build common values
+      timestamp = datetime.strftime(LOG_HEADER_TIME)
+
+      # Build header
+      header = sprintf LOG_HEADER_FORMAT,
+        timestamp,
+        Process.pid,
+        severity,
+        progname
+
+      # If we have a bunch of lines, prefix them and send them together
+      return payload.map do |line|
+        "#{header}#{trimmed(line)}\n"
+      end.join if payload.is_a?(Array)
+
+      # Otherwise, just prefix the only line
+      return "#{header}#{trimmed(payload)}\n"
+    end
+
+  protected
+
+    def self.trimmed line
+      line.to_s.rstrip[0..LOG_MESSAGE_TRIM].force_encoding(Encoding::UTF_8)
+    end
+
+  end
+end

data/lib/shared/logger_helper.rb

@@ -0,0 +1,78 @@
+require "logger"
+
+module Shared
+  module LoggerHelper
+
+  protected
+
+    def log_info message, details = nil
+      build_messages Logger::INFO, message, details
+    end
+
+    def log_error message, details = nil
+      build_messages Logger::ERROR, message, details
+    end
+
+    def log_debug message, details = nil
+      build_messages Logger::DEBUG, message, details
+    end
+
+    alias info log_info
+    alias error log_error
+    alias debug log_debug
+
+  private
+
+    # Builds prefix if LOG_PREFIX_FORMAT defined and caller has log_prefix method to provide values
+    def build_prefix
+      # Skip if no values from user class
+      return unless respond_to?(:log_prefix, true)
+      values = log_prefix
+
+      # Skip if no format defined
+      return unless defined?('LOG_PREFIX_FORMAT')
+      return unless LOG_PREFIX_FORMAT.is_a? String
+
+      # Build prefix string
+      LOG_PREFIX_FORMAT % values.map(&:to_s)
+    end
+
+    def build_messages severity, message, details = nil
+      messages = []
+      # messages << "/---------------------------------------"
+      # messages << "severity: #{severity}"
+      # messages << "message: #{message.class}"
+      # messages << "details: #{details.class} #{details.inspect}"
+      # messages << "ARRAY(#{details.count})" if details.is_a? Array
+      # messages << "HASH(#{details.count})" if details.is_a? Hash
+
+      prefix = build_prefix
+
+      # Add main message
+      messages << sprintf(LOG_MESSAGE_TEXT, prefix, message) if message
+
+      # Add details from array
+      details.each do |line|
+        messages << sprintf(LOG_MESSAGE_ARRAY, prefix, line)
+      end if details.is_a? Array
+
+      # Add details from hash
+      details.each do |key, value|
+        messages << sprintf(LOG_MESSAGE_HASH, prefix, key, value)
+      end if details.is_a? Hash
+
+      # Return all that stuff
+      # messages << "\\---------------------------------------"
+      logger.add severity, messages
+    end
+
+    # def debug_lines lines, prefix = ''
+    #   if lines.is_a? Array
+    #     logger.debug lines.map{ |line| sprintf(LOG_MESSAGE_ARRAY, prefix, line) }
+    #   elsif lines.is_a? Hash
+    #     logger.debug lines.map{ |key, value| sprintf(LOG_MESSAGE_HASH, prefix, key, value) }
+    #   end
+    # end
+
+  end
+end

data/rest-ftp-daemon.gemspec

@@ -1,40 +1,41 @@
 # coding: utf-8
+Gem::Specification.new do |spec|
 
-# Libs
-require_relative "lib/rest-ftp-daemon/constants"
-
+  # Project version
+  spec.version      = "0.300.1"
 
-# Gemspec
-Gem::Specification.new do |spec|
-  spec.name = APP_NAME
-  spec.date = Time.now.strftime("%Y-%m-%d")
-  spec.authors = ["Bruno MEDICI"]
-  spec.email = "rest-ftp-daemon@bmconseil.com"
-  spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
-  spec.summary = "RESTful FTP client daemon"
-  spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
-  spec.licenses = ["MIT"]
+  # Project description
+  spec.name                   = "rest-ftp-daemon"
+  spec.authors                = ["Bruno MEDICI"]
+  spec.email                  = "rest-ftp-daemon@bmconseil.com"
+  spec.description            = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
+  spec.summary                = "RESTful FTP client daemon"
+  spec.homepage               = "http://github.com/bmedici/rest-ftp-daemon"
+  spec.licenses               = ["MIT"]
+  spec.date                   = Time.now.strftime("%Y-%m-%d")
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f == "dashboard.png"
-  end
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-  spec.version       = APP_VER
+  # List files and executables
+  spec.files                  = `git ls-files -z`.split("\x0").reject{ |f| f == "dashboard.png"}
+  spec.executables            = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths          = ["lib"]
+  spec.required_ruby_version  = ">= 2.2"
 
-  spec.required_ruby_version = ">= 2.2"
 
+  # Development dependencies
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "http", "~> 0.8"
+  spec.add_development_dependency "http"
+  #spec.add_development_dependency "http", "~> 0.8"
   spec.add_development_dependency "rubocop", "~> 0.32.0"
   spec.add_development_dependency "pry"
 
-  spec.add_runtime_dependency "thin", "~> 1.6"
+  # Runtime dependencies
+  spec.add_runtime_dependency "thin", "~> 1"
   spec.add_runtime_dependency "grape"
   spec.add_runtime_dependency "grape-entity"
   spec.add_runtime_dependency "settingslogic"
+  spec.add_runtime_dependency "chamber"
   spec.add_runtime_dependency "haml"
   spec.add_runtime_dependency "json"
   spec.add_runtime_dependency "net-sftp"

data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml}

@@ -3,7 +3,8 @@ defaults: &defaults
   port: 3000
   user: rftpd
   group: rftpd
-  #host: "myhost"
+  # host: "myhost"
+  # pidfile: "/tmp/rftpd.pid"
 
   pools:
     default: 2
@@ -39,15 +40,17 @@ defaults: &defaults
     # clean_queued: 86400
 
   newrelic:
-    licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    licence: ""
+    # prefix: "rftpd"                       # app prefix
     platform: "bigbusiness"               # app platform name
-    app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
+    # app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
 
   debug:
-    # ftp: false
-    # sftp: false
-    # conchita: false
-    # allow_reload: false
+    ftp: false
+    sftp: false
+    conchita: false
+    reporter: fakse
+    allow_reload: false
 
   logs:
     thin:    "/var/log/rftpd-environment-thin.log"