rise-mirror_manager 0.1.0

data/lib/rise/mirror_manager/configuration.rb

@@ -0,0 +1,17 @@
+module Rise
+  module MirrorManager
+    class << self
+      attr_accessor :configuration
+    end
+
+    def self.configure
+      self.configuration ||= Configuration.new
+      yield(configuration)
+    end
+
+    # Class containing application configuration
+    class Configuration
+      attr_accessor :log_dir, :admin_email, :mirror_data_file, :database, :slack_webhook
+    end
+  end
+end

data/lib/rise/mirror_manager/database.rb

@@ -0,0 +1,49 @@
+require 'sqlite3'
+
+module Rise
+  module MirrorManager
+    # Module for database interaction
+    module Database
+      DB = SQLite3::Database.open(Rise::MirrorManager.configuration.database, results_as_hash: true)
+
+      def get_sync_status(mirror_name)
+        columns = %w[sync_status total_bandwidth error]
+        row = DB.execute("SELECT #{columns.join(',')} FROM sync_info WHERE mirror_id = (SELECT mirror_id FROM mirrors WHERE name = ?)", [mirror_name])[0] #Retrieve first row of the Array
+        if row.nil? || row.empty?
+          nil
+        else
+          return row.select { |key, _| columns.include?(key.to_s) }
+        end
+      end
+
+      def update_sync_status(mirror_name:, status:, total_bandwidth: nil, error: nil)
+        mirror_id = retrieve_mirror_id(mirror_name)
+        if sync_info_exists?(mirror_id)
+          DB.execute 'UPDATE sync_info SET sync_status =  ?, total_bandwidth = ?, error = ? WHERE mirror_id = ?', [status, total_bandwidth, error, mirror_id]
+        else
+          DB.execute 'INSERT INTO sync_info (mirror_id, sync_status, total_bandwidth, error) VALUES (?, ?, ?, ?)', [mirror_id, status, total_bandwidth, error]
+        end
+      end
+
+      private
+
+      def retrieve_mirror_id(mirror_name)
+        row = (DB.execute 'SELECT mirror_id FROM mirrors WHERE name = ?', [mirror_name])[0]
+        if row.nil? || row.empty?
+          raise "#{mirror_name} not in database."
+        else
+          row["mirror_id"]
+        end
+      end
+
+      def sync_info_exists?(mirror_id)
+        sync_info = DB.execute 'SELECT sync_info_id FROM sync_info WHERE mirror_id = ?', [mirror_id]
+        if sync_info.nil? || sync_info.empty?
+          false
+        else
+          true
+        end
+      end
+    end
+  end
+end

data/lib/rise/mirror_manager/error/cmd_error.rb

@@ -0,0 +1,15 @@
+require_relative 'custom_error'
+
+module Rise
+  module MirrorManager
+    module Error
+      # Custom exception class for cmd failures
+      class CmdError < CustomError
+        def initialize(msg, original = $ERROR_INFO)
+          super(:cmd_error, msg)
+          @original = original
+        end
+      end
+    end
+  end
+end

data/lib/rise/mirror_manager/error/custom_error.rb

@@ -0,0 +1,19 @@
+module Rise
+  module MirrorManager
+    module Error
+      # Class for RISE custom error exception
+      class CustomError < StandardError
+        attr_reader :status, :error, :message
+
+        def initialize(error = nil, msg = nil)
+          @error = error || :generic_error
+          @message = msg || 'Something went wrong'
+        end
+
+        def log_error
+          # log error
+        end
+      end
+    end
+  end
+end

data/lib/rise/mirror_manager/error/error_handler.rb

@@ -0,0 +1,23 @@
+module Rise
+  module MirrorManager
+    module Error
+      module ErrorHandler
+
+        def self.included(clazz)
+          clazz.class_eval do
+            rescue_from CmdError do |e|
+              #log error
+            end
+            rescue_from StandardError do |e|
+              #log error
+            end
+
+          end
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/rise/mirror_manager/error/lock_error.rb

@@ -0,0 +1,14 @@
+require_relative 'custom_error'
+
+module Rise
+  module MirrorManager
+    module Error
+      class LockError < CustomError
+        def initialize()
+          super(:lock_error, 
+                
+        end
+      end
+    end
+  end
+end

data/lib/rise/mirror_manager/logging.rb

@@ -0,0 +1,60 @@
+require_relative 'configuration'
+require 'logger'
+
+module Rise
+  module MirrorManager
+    # Module containing logging functionalities
+    module Logging
+      class << self
+        attr_reader :app_logger
+      end
+
+      def self.setup_logger
+        log_file = File.new(File.join(Rise::MirrorManager.configuration.log_dir, 'mirror_manager.log'), File::CREAT | File::WRONLY | File::APPEND)
+        log_file.sync = true
+        app_logger = Logger.new(log_file, 'weekly',
+                                 formatter: proc { |severity, datetime, _progname, msg|
+          "[#{Process.pid}][#{datetime}] #{severity} -- #{msg}\n"
+        })
+      end
+
+      # class IOToLog < IO
+      #   def initialize(logger)
+      #     @logger = logger
+      #   end
+
+      #   def write(message)
+      #     @logger.error message
+      #   end
+      # end
+
+      # Logger object for individual sync jobs
+      class SyncLog
+        def initialize(filename)
+          log_file = File.open(File.join(Rise::MirrorManager.configuration.log_dir,
+                                         "#{filename}.log"), File::CREAT | File::WRONLY | File::APPEND)
+          log_file.sync = true # flush logs in realtime
+          @logger = Logger.new(log_file, 'weekly', formatter: proc { |severity, datetime, _progname, msg|
+            "[#{Process.pid}:#{Thread.current.object_id}][#{datetime}] #{severity} -- #{msg}\n"
+          })
+        end
+
+        def log_info(msg)
+          logger.info msg
+        end
+
+        def log_warn(msg)
+          logger.warn msg
+        end
+
+        def log_error(msg)
+          logger.error msg
+        end
+
+        private
+
+        attr_reader :logger
+      end
+    end
+  end
+end

data/lib/rise/mirror_manager/mirror.rb

@@ -0,0 +1,46 @@
+require_relative 'configuration'
+require_relative 'logging'
+require 'ostruct'
+require 'inifile'
+
+module Rise
+  module MirrorManager
+    # Module for Mirror data
+    module Mirror
+      def self.retrieve_mirrors(interval)
+        mirrors = extract_mirrors(Rise::MirrorManager.configuration.mirror_data_file, interval)
+        if mirrors.empty?
+          raise "No mirror was retrieved for interval: #{interval}."
+        end
+        mirrors
+      end
+
+      def self.extract_mirrors(mirror_data_file, interval)
+        data_file = IniFile.load(mirror_data_file)
+        if data_file.nil?
+          raise "#{mirror_data_file} not found."
+        end
+        mirrors = []
+        data_file.each_section do |section|
+          next if data_file[section]['INTERVAL'] != interval
+          mirror = OpenStruct.new(name: data_file[section]['NAME'])
+          if data_file[section]['CUSTOM_SCRIPT']
+            mirror[:custom_script] = data_file[section]['CUSTOM_SCRIPT']
+          else
+            mirror[:remote_source] = data_file[section]['REMOTE_SOURCE']
+            mirror[:local_dir] = data_file[section]['LOCAL_DIR']
+            mirror[:rsync_opts] = data_file[section]['RSYNC_OPTS']
+            mirror[:rsync_del_opts] = data_file[section]['RSYNC_DEL_OPTS']
+            mirror[:exclude] = data_file[section]['EXCLUDE']
+            mirror[:include] = data_file[section]['INCLUDE']
+            mirror[:filter] = data_file[section]['FILTER']
+            mirror[:precmd] = data_file[section]['PRECMD']
+            mirror[:postcmd] = data_file[section]['POSTCMD']
+            mirror[:depth] = data_file[section]['DEPTH']
+          end
+          mirrors << mirror
+        end
+        mirrors
+      end end
+  end
+end

data/lib/rise/mirror_manager/notification.rb

@@ -0,0 +1,46 @@
+require_relative 'configuration'
+require 'pony'
+require 'slack/incoming/webhooks'
+
+module Rise
+  module MirrorManager
+    # Module containing notification functionalities
+    module Notification
+      SLACK = Slack::Incoming::Webhooks.new Rise::MirrorManager.configuration.slack_webhook
+
+      def send_email(msg)
+        Pony.mail({
+          :to => Rise::MirrorManager.configuration.admin_email,
+          :from => "MirrorManager@#{Socket.gethostname}",
+          :via => :smtp,
+          :via_options  =>  {
+            :address    =>  'aspmx.l.google.com',
+            :port       =>  25,
+            :domain     =>  Socket.gethostname
+          },
+          :subject      => "[#{Socket.gethostname}]MirrorManager Notification @ #{(Time.now).strftime("[%Y-%m-%d]%A, %H:%M")}",
+          :body         => msg
+        })
+      end
+
+      def send_slack_message(mirror_name:, error:, runtime:)
+        attachment = {
+          fallback: "#{mirror_name} has failed syncing @ #{runtime}. Error: #{error}",
+          color: "#FF0000",
+          "fields": [
+            {
+              "title": "#{mirror_name} Failed",
+              "value": "#{error}"
+            }
+          ],
+          "footer": "#{runtime}"
+        }
+
+        SLACK.post '', attachments: [attachment]
+        puts 'hmmm'
+        # https://hooks.slack.com/services/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
+        # TODO: IMplement
+      end
+    end
+  end
+end

data/lib/rise/mirror_manager/sync.rb

@@ -0,0 +1,196 @@
+# require File.join(__dir__, '../../../config/initializers/production')
+require_relative 'error/cmd_error'
+require_relative 'logging'
+require_relative 'database'
+require_relative 'sync_status'
+require 'marz/rsync'
+require 'English'
+require 'tmpdir'
+require 'ostruct'
+require 'open3'
+
+module Rise
+  module MirrorManager
+    # This will create an instance
+    # for every mirror rsync job
+    class Sync
+      include Rise::MirrorManager::Error
+      include Rise::MirrorManager::Logging
+      include Rise::MirrorManager::SyncStatus
+      include Rise::MirrorManager::Database
+
+      DEFAULT_RSYNC_OPTIONS = '-avzuH --no-motd --safe-links --numeric-ids' \
+        ' --delay-updates --timeout=600 --contimeout=300'.freeze
+      DEFAULT_RSYNC_DELETE_OPTIONS = '--delete --delete-delay'.freeze
+
+      def self.run(mirror)
+        Sync.new(mirror).run
+      end
+
+      def run
+        err = catch :failure do
+          if mirror.custom_script
+            execute_cmd(mirror.custom_script) { update_sync_status(mirror_name: mirror.name, status: SUCCESS) }
+            # execute_custom_script(mirror.custom_script)
+          else
+            begin
+              # err = catch :failure do
+              @rsync_opts = build_rsync_opts(mirror.rsync_opts, mirror.rsync_delete_opts)
+              # Can't capture the error `File.join(mirror.local_dir, "#{mirror.name}.lock")` when mirror.local_dir == nil
+              @lock_file = File.join(mirror.local_dir, "#{mirror.name}.lock")
+              establish_lock
+              # execute_precmd(mirror.pre_cmd) if mirror.pre_cmd
+              execute_cmd(mirror.pre_cmd) if mirror.pre_cmd
+              sync
+              # execute_postcmd(mirror.post_cmd) if mirror.post_cmd
+              execute_cmd(mirror.post_cmd) if mirror.post_cmd
+              update_sync_status(mirror_name: mirror.name, status: SUCCESS, total_bandwidth: total_bandwidth)
+            ensure
+              release_lock
+            end
+          end
+        end
+        update_sync_status(mirror_name: mirror.name, status: FAILED, error_msg: err) if err
+      end
+
+    private
+
+    def initialize(mirror)
+      @mirror = mirror
+      @logger = SyncLog.new(mirror.name)
+    end
+
+
+    attr_reader :mirror, :lock_file, :rsync_opts, :logger, :total_bandwidth
+
+    def sync
+      Marz::Rsync.run(mirror.remote_source, mirror.local_dir, rsync_opts) do |result|
+        if result.success?
+          @total_bandwidth = convert_bytes(result.total_bytes_sent + result.total_bytes_received)
+          logger.log_info result.output
+        else
+          logger.log_error result.output
+          throw :failure, result.output
+        end
+      end
+    end
+
+    def execute_cmd(cmd)
+      Open3.popen2e(cmd.to_s) do |_i, oe, t|
+        unless t.value.success?
+          throw :failure, oe.read
+        end
+      end
+      yield if block_given?
+    end
+
+    # def execute_precmd(cmd)
+    #   Open3.popen2e(cmd.to_s) do |_i, oe, t|
+    #     unless t.value.success?
+    #       update_precmd_result(mirror.name, FAILED)
+    #       msg = oe.read # Can only be retrieved/read once
+    #       logger.log_error(msg) unless msg.nil? || msg.empty?
+    #       exit
+    #     end
+    #     update_precmd_result(mirror.name, SUCCESS)
+    #   end
+    # rescue Errno::ENOENT => e
+    #   update_precmd_result(mirror.name, FAILED)
+    #   logger.log_error(e.message) unless e.message.nil? || e.message.empty?
+    #   exit
+    # end
+
+    # def execute_postcmd(cmd)
+    #   Open3.popen2e(cmd.to_s) do |_i, oe, t|
+    #     unless t.value.success?
+    #       update_postcmd_result(mirror.name, FAILED)
+    #       msg = oe.read # Can only be retrieved/read once
+    #       logger.log_error(msg) unless msg.nil? || msg.empty?
+    #       exit
+    #     end
+    #     update_postcmd_result(mirror.name, SUCCESS)
+    #   end
+    # rescue Errno::ENOENT => e
+    #   update_postcmd_result(mirror.name, FAILED)
+    #   logger.log_error(e.message) unless e.message.nil? || e.message.empty?
+    #   exit
+    # end
+
+    # def execute_custom_script(cmd)
+    #   execute_cmd(cmd) do |_oe, t|
+    #     update_sync_status(mirror.name, SUCCESS) if t.value.success?
+    #   end
+    # end
+
+    # def execute_cmd(cmd)
+    #   Open3.popen2e(cmd.to_s) do |_i, oe, t|
+    #     yield(oe, t) if block_given?
+    #     process_sync_failure(oe.read) unless t.value.success?
+    #   end
+    # rescue Errno::ENOENT => e
+    #   process_sync_failure(e.message)
+    # end
+
+    def process_sync_failure(msg = nil)
+      update_sync_status(mirror.name, FAILED)
+      logger.log_error(msg) unless msg.nil? || msg.empty?
+      exit
+    end
+
+    def establish_lock
+      if File.exist?(lock_file)
+        acquire_lock
+      else
+        create_lock
+      end
+    end
+
+    def acquire_lock
+      pid = File.read(lock_file).to_i
+      Process.kill(0, pid)
+    rescue Errno::ESRCH
+      File.delete(lock_file) # nonexistent process
+      create_lock
+    rescue Errno::EPERM
+      throw :failure, "No permission/privilege to send signal to process(#{pid})"
+    else
+      throw :failure, "Cannot establish lock. Another process(#{pid}) is currently running and holding the lock."
+    end
+
+    def create_lock
+      File.open(lock_file, File::RDWR | File::CREAT, 0o644) do |f|
+        f.flock(File::LOCK_EX)
+        f.write(Process.pid.to_s)
+        f.flush
+      end
+    end
+
+    def release_lock
+      File.delete(lock_file) if File.exist?(lock_file)
+    end
+
+    def build_rsync_opts(rsync_opts, rsync_delete_opts)
+      opts = rsync_opts || DEFAULT_RSYNC_OPTIONS
+      opts += ' ' + (rsync_delete_opts || DEFAULT_RSYNC_DELETE_OPTIONS)
+      opts.split(' ')
+    end
+
+    def convert_bytes(bytes)
+      bytes = Integer(bytes)
+      result =
+        if bytes <= 104_857_0
+          format('%.2fK', (bytes / 1024.0))
+        elsif bytes <= 107_373_645_5 # 0.99GB
+          format('%.2fM', ((bytes / 1024.0) / 1024.0))
+        else
+          format('%.2fG', (((bytes / 1024.0) / 1024.0) / 1024.0))
+        end
+      result.gsub(/(\.)0+/, '')
+    end
+
+  end
+end
+end
+
+# mirror = OpenStruct.new(name:"Delete Me", remote_source:"/tmp/srv/mirror/delete-me-source", local_dir:"/tmp/srv/mirror/delete-me-dest", rsync_opts:'-avzuH --no-motd --safe-links --numeric-ids --delay-updates --timeout=600', rsync_del_opts:nil, exclude:nil, include:nil, filter:nil, precmd:nil, postcmd:nil, depth:1)
+# Rise::MirrorManager::Sync.new(mirror).run