rest-ftp-daemon 0.302.3 → 0.304.0

data/lib/rest-ftp-daemon.rb

@@ -1,6 +1,6 @@
-
 # Global libs
 require "rubygems"
+require 'bmc-daemon-lib'
 require "json"
 require "haml"
 require "uri"
@@ -10,32 +10,20 @@ require "thread"
 require "newrelic_rpm"
 
 
-# Shared libs
-require_relative "shared/logger_formatter"
-require_relative "shared/logger_helper"
-require_relative "shared/conf"
-require_relative "shared/worker_base"
-
-
-# HAML and Array monkey-patching
-require_relative "rest-ftp-daemon/patch_array"
-require_relative "rest-ftp-daemon/patch_haml"
+# Shared libs / monkey-patching
+require_relative "shared/patch_array"
+require_relative "shared/patch_haml"
 
 
 # Project's libs
 require_relative "rest-ftp-daemon/constants"
 require_relative "rest-ftp-daemon/exceptions"
 require_relative "rest-ftp-daemon/helpers"
-require_relative "rest-ftp-daemon/logger_pool"
 require_relative "rest-ftp-daemon/metrics"
 require_relative "rest-ftp-daemon/paginate"
 require_relative "rest-ftp-daemon/uri"
 require_relative "rest-ftp-daemon/job_queue"
 require_relative "rest-ftp-daemon/counters"
-require_relative "rest-ftp-daemon/worker_pool"
-require_relative "rest-ftp-daemon/workers/conchita"
-require_relative "rest-ftp-daemon/workers/reporter"
-require_relative "rest-ftp-daemon/workers/transfer"
 require_relative "rest-ftp-daemon/job"
 require_relative "rest-ftp-daemon/notification"
 
@@ -44,6 +32,12 @@ require_relative "rest-ftp-daemon/remote"
 require_relative "rest-ftp-daemon/remote_ftp"
 require_relative "rest-ftp-daemon/remote_sftp"
 
+require_relative "rest-ftp-daemon/worker_pool"
+require_relative "rest-ftp-daemon/workers/worker"
+require_relative "rest-ftp-daemon/workers/conchita"
+require_relative "rest-ftp-daemon/workers/reporter"
+require_relative "rest-ftp-daemon/workers/transfer"
+
 require_relative "rest-ftp-daemon/api/job_presenter"
 require_relative "rest-ftp-daemon/api/jobs"
 require_relative "rest-ftp-daemon/api/dashboard"

data/lib/{rest-ftp-daemon → shared}/patch_haml.rb

@@ -1,10 +1,8 @@
 module Haml
   class Buffer
     alias_method :haml_push_text, :push_text
-
     def push_text text, tab_change, dont_tab_up
       haml_push_text text.force_encoding("utf-8"), tab_change, dont_tab_up
     end
-
   end
 end

data/rest-ftp-daemon.gemspec

@@ -2,24 +2,24 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version      = "0.302.3"
+  spec.version                    = "0.304.0"
 
   # Project description
-  spec.name                   = "rest-ftp-daemon"
-  spec.authors                = ["Bruno MEDICI"]
-  spec.email                  = "rest-ftp-daemon@bmconseil.com"
-  spec.description            = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
-  spec.summary                = "RESTful FTP client daemon"
-  spec.homepage               = "http://github.com/bmedici/rest-ftp-daemon"
-  spec.licenses               = ["MIT"]
-  spec.date                   = Time.now.strftime("%Y-%m-%d")
+  spec.name                       = "rest-ftp-daemon"
+  spec.authors                    = ["Bruno MEDICI"]
+  spec.email                      = "rest-ftp-daemon@bmconseil.com"
+  spec.description                = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
+  spec.summary                    = "RESTful FTP client daemon"
+  spec.homepage                   = "http://github.com/bmedici/rest-ftp-daemon"
+  spec.licenses                   = ["MIT"]
+  spec.date                       = Time.now.strftime("%Y-%m-%d")
 
 
   # List files and executables
-  spec.files                  = `git ls-files -z`.split("\x0").reject{ |f| f == "dashboard.png"}
-  spec.executables            = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.require_paths          = ["lib"]
-  spec.required_ruby_version  = ">= 2.2.3"
+  spec.files                      = `git ls-files -z`.split("\x0").reject{ |f| f == "dashboard.png"}
+  spec.executables                = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths              = ["lib"]
+  spec.required_ruby_version      = ">= 2.2"
 
 
   # Development dependencies
@@ -28,21 +28,22 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rubocop", "~> 0.32.0"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "http"
 
   # Runtime dependencies
-  spec.add_runtime_dependency "json", "~> 1.8"
-  spec.add_runtime_dependency "thin", "~> 1.7"
-  spec.add_runtime_dependency "grape"
-  spec.add_runtime_dependency "grape-entity"
-  spec.add_runtime_dependency "settingslogic"
-  spec.add_runtime_dependency "chamber"
-  spec.add_runtime_dependency "rest-client", "~> 1.8"
-  spec.add_runtime_dependency "api-auth"
-  spec.add_runtime_dependency "haml"
-  spec.add_runtime_dependency "net-sftp"
-  spec.add_runtime_dependency "double-bag-ftps"
-  spec.add_runtime_dependency "facter"
-  spec.add_runtime_dependency "sys-cpu"
-  spec.add_runtime_dependency "get_process_mem"
-  spec.add_runtime_dependency "newrelic_rpm"
+  spec.add_runtime_dependency     "bmc-daemon-lib", "~> 0.2"
+  spec.add_runtime_dependency     "json", "~> 1.8"
+  spec.add_runtime_dependency     "thin", "~> 1.7"
+  spec.add_runtime_dependency     "grape"
+  spec.add_runtime_dependency     "grape-entity"
+  spec.add_runtime_dependency     "settingslogic"
+  spec.add_runtime_dependency     "rest-client", "~> 1.8"
+  spec.add_runtime_dependency     "api-auth"
+  spec.add_runtime_dependency     "haml"
+  spec.add_runtime_dependency     "net-sftp"
+  spec.add_runtime_dependency     "double-bag-ftps"
+  spec.add_runtime_dependency     "facter"
+  spec.add_runtime_dependency     "sys-cpu"
+  spec.add_runtime_dependency     "get_process_mem"
+  spec.add_runtime_dependency     "newrelic_rpm"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.302.3
+  version: 0.304.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-12 00:00:00.000000000 Z
+date: 2016-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,49 +81,63 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bmc-daemon-lib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
-  name: thin
+  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: grape
+  name: thin
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
-  name: grape-entity
+  name: grape
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: settingslogic
+  name: grape-entity
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -151,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: chamber
+  name: settingslogic
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -325,12 +339,9 @@ files:
 - lib/rest-ftp-daemon/job.rb
 - lib/rest-ftp-daemon/job_queue.rb
 - lib/rest-ftp-daemon/launcher.rb
-- lib/rest-ftp-daemon/logger_pool.rb
 - lib/rest-ftp-daemon/metrics.rb
 - lib/rest-ftp-daemon/notification.rb
 - lib/rest-ftp-daemon/paginate.rb
-- lib/rest-ftp-daemon/patch_array.rb
-- lib/rest-ftp-daemon/patch_haml.rb
 - lib/rest-ftp-daemon/path.rb
 - lib/rest-ftp-daemon/remote.rb
 - lib/rest-ftp-daemon/remote_ftp.rb
@@ -351,10 +362,9 @@ files:
 - lib/rest-ftp-daemon/workers/conchita.rb
 - lib/rest-ftp-daemon/workers/reporter.rb
 - lib/rest-ftp-daemon/workers/transfer.rb
-- lib/shared/conf.rb
-- lib/shared/logger_formatter.rb
-- lib/shared/logger_helper.rb
-- lib/shared/worker_base.rb
+- lib/rest-ftp-daemon/workers/worker.rb
+- lib/shared/patch_array.rb
+- lib/shared/patch_haml.rb
 - rest-ftp-daemon.gemspec
 - spec/rest-ftp-daemon/features/dashboard_spec.rb
 - spec/rest-ftp-daemon/features/debug_spec.rb
@@ -374,7 +384,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.3
+      version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -1,61 +0,0 @@
-require "logger"
-require "singleton"
-
-# Logger interface class to access logger though symbolic names
-module RestFtpDaemon
-  class LoggerPool
-    include Singleton
-
-    def get pipe
-      @loggers ||= {}
-      @loggers[pipe] ||= create(pipe)
-    end
-
-    def create pipe
-      # Compute logfile or STDERR, and declare what we're doing
-      filename = logfile(pipe)
-
-      # Create the logger and return it
-      logger = Logger.new(filename, LOG_ROTATION)   #, 10, 1024000)
-      logger.progname = pipe.to_s.downcase
-      logger.formatter = Shared::LoggerFormatter
-
-      # Finally return this logger
-      logger
-
-    rescue Errno::EACCES
-      puts "LoggerPool [#{pipe}] failed: access error"
-    end
-
-  protected
-
-    def logfile pipe
-      # Disabled if no valid config
-      return nil unless Conf[:logs].is_a?(Hash)
-
-      # Compute logfile and check if we can write there
-      logfile = File.expand_path(Conf[:logs][pipe].to_s, Conf[:logs][:path].to_s)
-
-      # Check that we'll be able to create logfiles
-      if File.exists?(logfile)
-        # File is there, is it writable ?
-        unless File.writable?(logfile)
-          puts "LoggerPool [#{pipe}] disabled: file not writable [#{logfile}]"
-          return nil
-        end
-      else
-        # No file here, can we create it ?
-        logdir = File.dirname(logfile)
-        unless File.writable?(logdir)
-          puts "LoggerPool [#{pipe}] disabled: directory not writable [#{logdir}]"
-          return nil
-        end
-      end
-
-      # OK, return a clean file path
-      puts "LoggerPool [#{pipe}] logging to [#{logfile}]"
-      return logfile
-    end
-
-  end
-end

data/lib/shared/conf.rb

@@ -1,201 +0,0 @@
-# FIXME: files named with hyphens will not be found by Chamber for now
-require "chamber"
-
-module Shared
-  class ConfigMissingParameter    < StandardError; end
-  class ConfigOtherError          < StandardError; end
-  class ConfigParseError          < StandardError; end
-  class ConfigMultipleGemspec     < StandardError; end
-  class ConfigMissingGemspec      < StandardError; end
-
-  class Conf
-    extend Chamber
-    PIDFILE_DIR = "/tmp/"
-
-    class << self
-      attr_accessor :app_env
-      attr_reader   :app_root
-      attr_reader   :app_libs
-      attr_reader   :app_name
-      attr_reader   :app_ver
-      attr_reader   :app_started
-      attr_reader   :app_spec
-      attr_reader   :files
-      attr_reader   :host
-    end
-
-    def self.init app_root
-      # Permanent flags
-      @initialized  = true
-      @app_started  = Time.now
-
-      # Default values
-      @files        ||= []
-      @app_name     ||= "app_name"
-      @app_env      ||= "production"
-      @host         ||= `hostname`.to_s.chomp.split(".").first
-
-      # Store and clean app_root
-      @app_root = File.expand_path(app_root)
-
-      # Try to find any gemspec file
-      matches   = Dir["#{@app_root}/*.gemspec"]
-      fail ConfigMissingGemspec, "gemspec file not found: #{gemspec_path}" if matches.size < 1
-      fail ConfigMultipleGemspec, "gemspec file not found: #{gemspec_path}" if matches.size > 1
-
-      # Load Gemspec (just the only match)
-      @spec     = Gem::Specification::load(matches.first)
-      @app_name = @spec.name
-      @app_ver  = @spec.version
-      fail ConfigMissingParameter, "gemspec: missing name" unless @app_name
-      fail ConfigMissingParameter, "gemspec: missing version" unless @app_ver
-
-      # Now we know app_name, initalize app_libs
-      @app_libs = File.expand_path("lib/#{@app_name}/", @app_root)
-
-      # Add other config files
-      #add_default_config
-      add_config generate(:config_defaults)
-      add_config generate(:config_etc)
-
-      # Return something
-      return @app_name
-    end
-
-    def self.prepare args = {}
-      ensure_init
-
-      # Add extra config file and load them all
-      add_config args[:config]
-      reload!
-
-      # Set Rack env
-      ENV["RACK_ENV"] = @app_env.to_s
-
-      # Set up encodings
-      Encoding.default_internal = "utf-8"
-      Encoding.default_external = "utf-8"
-
-      # Init New Relic
-      newrelic_logfile = File.expand_path(Conf[:logs][:newrelic].to_s, Conf[:logs][:path].to_s)
-      prepare_newrelic self[:newrelic], newrelic_logfile
-
-      # Try to access any key to force parsing of the files
-      self[:dummy]
-
-    rescue Psych::SyntaxError => e
-      fail ConfigParseError, e.message
-    rescue StandardError => e
-      fail ConfigOtherError, "#{e.message} \n #{e.backtrace.to_yaml}"
-    end
-
-    # Reload files
-    def self.reload!
-      ensure_init
-      load_files
-    end
-
-    def self.dump
-      ensure_init
-      to_hash.to_yaml(indent: 4, useheader: true, useversion: false )
-    end
-
-    # Direct access to any depth
-    def self.at *path
-      ensure_init
-      path.reduce(Conf) { |m, key| m && m[key.to_s] }
-    end
-
-    def self.newrelic_enabled?
-      ensure_init
-      self[:newrelic] && self[:newrelic][:licence]
-    end
-
-    # Defaults generators
-    def self.generate what
-      ensure_init
-      return case what
-
-      when :user_agent
-        "#{@app_name}/#{@app_ver}" if @app_name && @app_ver
-
-      when :config_defaults
-        "#{@app_root}/defaults.yml" if @app_root
-
-      when :config_etc
-        "/etc/#{@app_name}.yml" if @app_name
-
-      when :process_name
-        parts = [@app_name, @app_env]
-        parts << self[:port] if self[:port]
-        parts.join('-')
-
-      when :pidfile
-        process_name = self.generate(:process_name)
-        File.expand_path "#{process_name}.pid", PIDFILE_DIR
-
-      when :config_message
-        config_defaults = self.generate(:config_defaults)
-        config_etc = self.generate(:config_etc)
-
-        "A default configuration is available (#{config_defaults}) and can be copied to the default location (#{config_etc}): \n sudo cp #{config_defaults} #{config_etc}"
-
-      end
-    end
-
-
-  protected
-
-    def self.load_files
-      load files: @files, namespaces: { environment: @app_env }
-    end
-
-    def self.add_config path
-      @files << File.expand_path(path) if path && File.readable?(path)
-    end
-
-    def self.prepare_newrelic section, logfile
-      # Disable NewRelic if no config present
-      unless self.newrelic_enabled?
-        ENV["NEWRELIC_AGENT_ENABLED"] = "false"
-        return
-      end
-
-      # Enable GC profiler
-      GC::Profiler.enable
-
-      # Enable module
-      ENV["NEWRELIC_AGENT_ENABLED"] = "true"
-      ENV["NEW_RELIC_MONITOR_MODE"] = "true"
-
-      # License
-      ENV["NEW_RELIC_LICENSE_KEY"] = section[:licence].to_s
-
-      # Build NewRelic app_name if not provided as-is
-      if section[:app_name]
-        ENV["NEW_RELIC_APP_NAME"] = section[:app_name].to_s
-      else
-        stack = []
-        stack << (section[:prefix] || @app_name)
-        stack << section[:platform] if section[:platform]
-        stack << @app_env
-        text = stack.join('-')
-        ENV["NEW_RELIC_APP_NAME"] = "#{text}-#{host};#{text}"
-      end
-
-      # Logfile
-      ENV["NEW_RELIC_LOG"] = logfile.to_s if logfile
-    end
-
-  private
-
-    def self.ensure_init
-      # Skip is already done
-      return if @initialized
-
-      # Go through init if not already done
-      self.init
-    end
-
-  end
-end

data/lib/shared/logger_formatter.rb

@@ -1,31 +0,0 @@
-module Shared
-  class LoggerFormatter
-
-    def self.call severity, datetime, progname, payload
-      # Build common values
-      timestamp = datetime.strftime(LOG_HEADER_TIME)
-
-      # Build header
-      header = sprintf LOG_HEADER_FORMAT,
-        timestamp,
-        Process.pid,
-        severity,
-        progname
-
-      # If we have a bunch of lines, prefix them and send them together
-      return payload.map do |line|
-        "#{header}#{trimmed(line)}\n"
-      end.join if payload.is_a?(Array)
-
-      # Otherwise, just prefix the only line
-      return "#{header}#{trimmed(payload)}\n"
-    end
-
-  protected
-
-    def self.trimmed line
-      line.to_s.rstrip[0..LOG_MESSAGE_TRIM].force_encoding(Encoding::UTF_8)
-    end
-
-  end
-end

data/lib/shared/logger_helper.rb

@@ -1,78 +0,0 @@
-require "logger"
-
-module Shared
-  module LoggerHelper
-
-  protected
-
-    def log_info message, details = nil
-      build_messages Logger::INFO, message, details
-    end
-
-    def log_error message, details = nil
-      build_messages Logger::ERROR, message, details
-    end
-
-    def log_debug message, details = nil
-      build_messages Logger::DEBUG, message, details
-    end
-
-    alias info log_info
-    alias error log_error
-    alias debug log_debug
-
-  private
-
-    # Builds prefix if LOG_PREFIX_FORMAT defined and caller has log_prefix method to provide values
-    def build_prefix
-      # Skip if no values from user class
-      return unless respond_to?(:log_prefix, true)
-      values = log_prefix
-
-      # Skip if no format defined
-      return unless defined?('LOG_PREFIX_FORMAT')
-      return unless LOG_PREFIX_FORMAT.is_a? String
-
-      # Build prefix string
-      LOG_PREFIX_FORMAT % values.map(&:to_s)
-    end
-
-    def build_messages severity, message, details = nil
-      messages = []
-      # messages << "/---------------------------------------"
-      # messages << "severity: #{severity}"
-      # messages << "message: #{message.class}"
-      # messages << "details: #{details.class} #{details.inspect}"
-      # messages << "ARRAY(#{details.count})" if details.is_a? Array
-      # messages << "HASH(#{details.count})" if details.is_a? Hash
-
-      prefix = build_prefix
-
-      # Add main message
-      messages << sprintf(LOG_MESSAGE_TEXT, prefix, message) if message
-
-      # Add details from array
-      details.each do |line|
-        messages << sprintf(LOG_MESSAGE_ARRAY, prefix, line)
-      end if details.is_a? Array
-
-      # Add details from hash
-      details.each do |key, value|
-        messages << sprintf(LOG_MESSAGE_HASH, prefix, key, value)
-      end if details.is_a? Hash
-
-      # Return all that stuff
-      # messages << "\\---------------------------------------"
-      logger.add severity, messages
-    end
-
-    # def debug_lines lines, prefix = ''
-    #   if lines.is_a? Array
-    #     logger.debug lines.map{ |line| sprintf(LOG_MESSAGE_ARRAY, prefix, line) }
-    #   elsif lines.is_a? Hash
-    #     logger.debug lines.map{ |key, value| sprintf(LOG_MESSAGE_HASH, prefix, key, value) }
-    #   end
-    # end
-
-  end
-end