elastic_manager 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59956a9221433200e4da7152617c039ac1dde775d30299876b6bb379a2358fde
-  data.tar.gz: c26c03c09b73188ef4e833a580981f538949918538e7df072f74a6eb5d55fd47
+  metadata.gz: 94343d3bac2671d6a49463c67d406afb79738edd347b63899ae32bcb58b1ac5e
+  data.tar.gz: aa36442960604bf810217e82d18c927db87d457e4d4c608202d85870ed190673
 SHA512:
-  metadata.gz: a4b322f736dcc9fea634113b7f661e49827e8baf26ccdcbd0e8f0c40c2e44c21fc5bf146abf04a751f8321fa157156f5a5fa4f6bfd6165edc89f96a3604103af
-  data.tar.gz: 6a06140123e5dc85b40b21ce47d115982c7293d6518ace6332614fcbba7e6a02b6d5b31636505d16a10a88c4bdb1210a140aa0612224e1a130fe164a81d4bb5c
+  metadata.gz: 892bf769d7e3392b4ce3dea9d3062b4ee3c2d7bb0335008478a6444d0b51c61d872b0d202ded7e978c854533b2150f08e560dacbf7dc04eb514d4bc9225358ef
+  data.tar.gz: aadad9042c0bd64848daf39c746639d3e8edcc0ad681686941ecd413324cba29375fdb1c586423abc727fef2e04cbaffe64b171f93c3cb1729640bf2763e0563

data/bin/elastic_manager

@@ -6,4 +6,4 @@ STDOUT.sync = true
 
 elastic_manager = ElasticManager.new(ARGV)
 
-elastic_manager.work
+elastic_manager.run

data/elastic_manager.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name        = 'elastic_manager'
   s.executables = ['elastic_manager']
-  s.version     = '0.1.3'
+  s.version     = '0.1.4'
   s.date        = '2018-10-15'
   s.summary     = 'Because qurator sucks'
   s.description = 'Manager for logstash indices in elastic'

data/lib/elastic_manager/close.rb

@@ -0,0 +1,18 @@
+module Close
+  def do_close(indices, date)
+    indices.each do |index_name|
+      next if skip?('close', index_name)
+
+      index    = "#{index_name}-#{date}"
+      response = @elastic.request(:get, "/_cat/indices/#{index}")
+
+      if index_exist?(response)
+        next if already?('close', response, index)
+
+        action_with_log('close', index)
+      else
+        log.warn "#{index} index not found, maybe already snapshoted"
+      end
+    end
+  end
+end

data/lib/elastic_manager/config.rb

@@ -35,15 +35,6 @@ module Config
     default
   end
 
-  def parse_settings(json)
-    begin
-      JSON.parse(json)
-    rescue JSON::ParserError => e
-      log.fatal "json parse err: '''#{e.message}'''\n\t#{e.backtrace.join("\n\t")}"
-      exit 1
-    end
-  end
-
   def option_parser(result)
     OptionParser.new do |parser|
       MAIN_PARAMS.each do |param|
@@ -60,7 +51,7 @@ module Config
             result[params[0].downcase][params[1].downcase] = pr
           elsif params.length == 1
             if params[0].downcase == 'settings'
-              result[params[0].downcase] = parse_settings(pr)
+              result[params[0].downcase] = json_parse(pr)
             else
               result[params[0].downcase] = pr
             end
@@ -72,46 +63,54 @@ module Config
     result
   end
 
-  def load_from_env
-    log.debug "will load config from ENV variables"
+  def get_env_vars(var, result)
+    vars = var.split('_')
 
-    result = make_default_config
+    if vars.length == 2
+      result[vars[0].downcase][vars[1].downcase] = ENV[var]
+    elsif vars.length == 1
+      if vars[0].downcase == 'settings'
+        result[vars[0].downcase] = json_parse(ENV[var])
+      else
+        result[vars[0].downcase] = ENV[var]
+      end
+    end
 
+    result
+  end
+
+  def env_parser(result)
     MAIN_PARAMS.each do |var|
       if ENV[var] == '' || ENV[var].nil?
         log.fatal BANNER_ENV
         exit 1
-      else
-        result[var.downcase] = ENV[var]
       end
+
+      result[var.downcase] = ENV[var]
     end
 
     ADDITIONAL_PARAMS.each do |var|
-      unless ENV[var] == '' || ENV[var].nil?
-        vars = var.split('_')
-
-        if vars.length == 2
-          result[vars[0].downcase][vars[1].downcase] = ENV[var]
-        elsif vars.length == 1
-          if vars[0].downcase == 'settings'
-            result[vars[0].downcase] = parse_settings(ENV[var])
-          else
-            result[vars[0].downcase] = ENV[var]
-          end
-        end
-      end
+      result = get_env_vars(var, result) unless ENV[var] == '' || ENV[var].nil?
     end
 
+    result
+  end
+
+  def load_from_env
+    log.debug "will load config from ENV variables"
+
+    result = make_default_config
+    result = env_parser(result)
+
     log.debug "env config: #{result.inspect}"
     result
   end
 
-  def load_from_argv(argv)
+  def load_from_argv
     require 'optparse'
 
     log.debug "will load config from passed arguments"
     result = make_default_config
-
     result = option_parser(result)
 
     if MAIN_PARAMS.map { |p| p.downcase }.map { |key| result[key].empty? }.any?{ |a| a == true }

data/lib/elastic_manager/logger.rb

@@ -2,6 +2,16 @@ require 'logger'
 require 'colorize'
 
 module Logging
+
+  SEVERITY_COLORS = {
+    'DEBUG'   => 'cyan',
+    'INFO'    => 'green',
+    'WARN'    => 'yellow',
+    'ERROR'   => 'light_red',
+    'FATAL'   => 'red',
+    'UNKNOWN' => 'magenta'
+  }
+
   def log
     @log ||= Logging.logger_for(self.class.name)
   end
@@ -15,35 +25,24 @@ module Logging
       @loggers[classname] ||= configure_logger_for(classname)
     end
 
-    def configure_logger_for(classname)
-      logger          = Logger.new(STDOUT)
-      logger.progname = classname
-
+    def log_level
       # :debug < :info < :warn < :error < :fatal < :unknown
       if ENV['LOG_LEVEL'] == '' || ENV['LOG_LEVEL'].nil?
-        logger.level = 'INFO'
+        'INFO'
       else
-        logger.level = ENV['LOG_LEVEL']
+        ENV['LOG_LEVEL']
       end
+    end
+
+    def configure_logger_for(classname)
+      logger          = Logger.new(STDOUT)
+      logger.progname = classname
+      logger.level    = log_level
 
       logger.formatter = proc do |severity, datetime, progname, msg|
         datetime = datetime.strftime("%Y-%m-%d | %I:%M:%S.%L")
-        message = "#{datetime} | #{progname} | #{severity} | #{msg}\n"
-
-        case severity
-        when 'DEBUG'
-          message.cyan
-        when 'INFO'
-          message.green
-        when 'WARN'
-          message.yellow
-        when 'ERROR'
-          message.light_red
-        when 'FATAL'
-          message.red
-        when 'UNKNOWN'
-          message.magenta
-        end
+        message  = "#{datetime} | #{progname} | #{severity} | #{msg}\n"
+        message.send(SEVERITY_COLORS[severity])
       end
 
       logger

data/lib/elastic_manager/open.rb

@@ -0,0 +1,21 @@
+module Open
+  def do_open(indices, date)
+    indices.each do |index_name|
+      next if skip?('open', index_name)
+
+      index    = "#{index_name}-#{date}"
+      response = @elastic.request(:get, "/_cat/indices/#{index}")
+
+      if index_exist?(response)
+        next if already?('open', response, index)
+
+        action_with_log('open', index)
+      else
+        log.warn "#{index} index not found"
+        log.info "#{index} trying snapshot restore"
+
+        action_with_log('restore_snapshot', index)
+      end
+    end
+  end
+end

data/lib/elastic_manager/request.rb

@@ -74,16 +74,18 @@ module Request
       indices.select!{ |_, v| v['state'] == state } if state
       indices.select!{ |_, v| v['settings']['index']['routing']['allocation']['require']['box_type'] == type } if type
 
-      indices.map do |index, _|
+      indices.select! do |index, _|
         begin
           index_date = Date.parse(index.gsub('-', ''))
         rescue ArgumentError => e
           log.error "#{e.message} for #{index}"
           next
         end
-
-        URI.escape(index) if (from..to).cover? index_date
+        (from..to).cover? index_date
       end
+
+      indices = indices.keys.map { |i| i.split('-')[0..-2].join('-') }
+      return indices
     end
 
     def get_all_indices
@@ -172,13 +174,13 @@ module Request
       true
     end
 
-    def open_index(index)
-      response = request(:post, "/#{index}/_open?master_timeout=1m")
+    def index(action, index)
+      response = request(:post, "/#{URI.escape(index)}/_#{action}?master_timeout=1m")
 
       if response.code == 200
         response = json_parse(response)
       else
-        log.fatal "wrong response code for #{index} open"
+        log.fatal "wrong response code for #{index} #{action}"
         exit 1
       end
 

data/lib/elastic_manager/utils.rb

@@ -4,6 +4,11 @@ require 'yajl'
 module Utils
   include Logging
 
+  REVERT_STATE = {
+    'open'  => 'close',
+    'close' => 'open'
+  }
+
   def true?(obj)
     obj.to_s.downcase == 'true'
   end
@@ -14,4 +19,87 @@ module Utils
     log.fatal "json parse err: '''#{e.message}'''\n\t#{e.backtrace.join("\n\t")}"
     exit 1
   end
+
+  def prechecks(date_from, date_to)
+    if date_from > date_to
+      log.fatal "wrong dates: date to is behind date from. from: #{date_from}, to: #{date_to}"
+      exit 1
+    end
+
+    unless true?(@config['force'])
+      unless @elastic.green?
+        log.fatal "elasticsearch on #{@config['es']['url']} is not green"
+        exit 1
+      end
+    end
+  end
+
+  def prepare_vars
+    indices   = @config['indices'].split(',')
+    date_from = Date.parse(@config['from'])
+    date_to   = Date.parse(@config['to'])
+
+    return indices, date_from, date_to
+  end
+
+  def all_precheck(indices, date_from, date_to, state)
+    if indices.length == 1 && indices.first == '_all'
+      indices = @elastic.all_indices(date_from, date_to, state)
+    end
+
+    indices
+  end
+
+  def action_with_log(action, index)
+    if @elastic.index(action, index)
+      log.info "#{index} #{action} succes"
+    else
+      log.error "#{index} #{action} fail"
+    end
+  end
+
+  def index_exist?(response)
+    if response.code == 200
+      return true
+    elsif response.code == 404
+      return false
+    else
+      log.fatal "wtf in index_exist? response was: #{response.code} - #{response}"
+      exit 1
+    end
+  end
+
+  def already?(status, response, index)
+    if json_parse(response).first['status'] == status
+      log.warn "#{index} index status already #{status}"
+      return true
+    end
+
+    false
+  end
+
+  def skip?(status, index_name)
+    if @config['settings'][index_name]
+      if @config['settings'][index_name]['skip'][status]
+        log.debug @config['settings'][index_name]['skip'][status].inspect
+
+        if true?(@config['settings'][index_name]['skip'][status])
+          log.warn "#{index_name} index #{status} skiped"
+          return true
+        end
+      end
+    end
+
+    false
+  end
+
+  def action(task)
+    indices, date_from, date_to = prepare_vars
+    prechecks(date_from, date_to)
+    indices = all_precheck(indices, date_from, date_to, REVERT_STATE[task])
+
+    date_from.upto(date_to) do |date|
+      self.send("do_#{task}", indices, date.to_s.tr!('-', '.'))
+    end
+  end
 end

data/lib/elastic_manager.rb

@@ -4,89 +4,28 @@ require 'elastic_manager/config'
 require 'elastic_manager/logger'
 require 'elastic_manager/request'
 require 'elastic_manager/utils'
+require 'elastic_manager/open'
+require 'elastic_manager/close'
 
 class ElasticManager
   include Config
   include Logging
   include Request
   include Utils
-
-  # attr_reader :config
+  include Open
+  include Close
 
   def initialize(argv)
     if argv.size == 0
       @config = load_from_env
     else
-      @config = load_from_argv(argv)
+      @config = load_from_argv
     end
 
     @elastic = Request::Elastic.new(@config)
   end
 
-  def work
-    if @config['task'].downcase == 'open'
-      indices   = @config['indices'].split(',')
-      date_from = Date.parse(@config['from'])
-      date_to   = Date.parse(@config['to'])
-
-      unless true?(@config['force'])
-        unless @elastic.green?
-          log.fatal "elasticsearch on #{@config['es']['url']} is not green"
-          exit 1
-        end
-      end
-
-      if indices.length == 1 && indices.first == '_all'
-        # indices = es_all_indices(date_from, date_to, 'close')
-        indices = @elastic.all_indices(date_from, date_to, 'close')
-      end
-
-      date_from.upto(date_to) do |date|
-        date = date.to_s.tr!('-', '.')
-
-        indices.each do |index_name|
-          if @config['settings'][index_name]
-            if @config['settings'][index_name]['skip_open']
-              log.debug @config['settings'][index_name]['skip_open'].inspect
-
-              if true?(@config['settings'][index_name]['skip_open'])
-                log.warn "#{index_name} index open skiped"
-                next
-              end
-            end
-          end
-
-          index = "#{index_name}-#{date}"
-
-          response = @elastic.request(:get, "/_cat/indices/#{index}")
-
-          if response.code == 404
-            log.warn "#{index} index not found"
-            log.info "#{index} trying snapshot restore"
-
-            if @elastic.restore_snapshot(index)
-              log.info "#{index} restored"
-            else
-              log.error "#{index} troubles with restore"
-            end
-          elsif response.code == 200
-            if response.body.to_s =~ /open/
-              log.warn "#{index} index already opened"
-              next
-            end
-
-            if @elastic.open_index(index)
-              log.info "#{index} index open success"
-            else
-              log.fatal "#{index} index open failed"
-              exit 1
-            end
-          else
-            log.fatal "can't work with index #{index} response was: #{response.code} - #{response}"
-            exit 1
-          end
-        end
-      end
-    end
+  def run
+    action(@config['task'].downcase)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_manager
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Antony Ryabov
@@ -94,8 +94,10 @@ files:
 - bin/elastic_manager
 - elastic_manager.gemspec
 - lib/elastic_manager.rb
+- lib/elastic_manager/close.rb
 - lib/elastic_manager/config.rb
 - lib/elastic_manager/logger.rb
+- lib/elastic_manager/open.rb
 - lib/elastic_manager/request.rb
 - lib/elastic_manager/utils.rb
 homepage: https://github.com/onetwotrip/elastic_manager