elastic_manager 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bca9616ef02571e4e12f488fb61367e5009788fa919d26dd862b3f5b11fa014f
-  data.tar.gz: 790823ce0244db16bf668e664a37bce2d644eda86f17283a1ae883242473aa78
+  metadata.gz: 59956a9221433200e4da7152617c039ac1dde775d30299876b6bb379a2358fde
+  data.tar.gz: c26c03c09b73188ef4e833a580981f538949918538e7df072f74a6eb5d55fd47
 SHA512:
-  metadata.gz: 4a25ec8e9155eaa37d6661828853c3708c1eb7bdf81b2f7fcd281eedc337d013e8bcf3d56cc245ce69f0208c7c9d921226260ef7472a765f07024591832a45ba
-  data.tar.gz: ffe738eae1d02696ec2516331f781326381495ab9d42ee30b83bb9e14d6670285d417e142f529f6b289eb819d62955062761670915cec9893a0d2897da49adbe
+  metadata.gz: a4b322f736dcc9fea634113b7f661e49827e8baf26ccdcbd0e8f0c40c2e44c21fc5bf146abf04a751f8321fa157156f5a5fa4f6bfd6165edc89f96a3604103af
+  data.tar.gz: 6a06140123e5dc85b40b21ce47d115982c7293d6518ace6332614fcbba7e6a02b6d5b31636505d16a10a88c4bdb1210a140aa0612224e1a130fe164a81d4bb5c

data/README.md

@@ -0,0 +1 @@
+[![Build Status](https://travis-ci.org/onetwotrip/elastic_manager.svg?branch=master)](https://travis-ci.org/onetwotrip/elastic_manager)

data/elastic_manager.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name        = 'elastic_manager'
   s.executables = ['elastic_manager']
-  s.version     = '0.1.2'
+  s.version     = '0.1.3'
   s.date        = '2018-10-15'
   s.summary     = 'Because qurator sucks'
   s.description = 'Manager for logstash indices in elastic'
@@ -12,7 +12,9 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/onetwotrip/elastic_manager'
   s.license     = 'MIT'
   s.required_ruby_version = '>= 2.0.0'
-  s.add_dependency 'http',     '~> 3.3'
-  s.add_dependency 'colorize', '~> 0.8'
-  s.add_dependency 'dotenv',   '~> 2.4'
+  s.add_dependency 'http',      '~> 3.3'
+  s.add_dependency 'colorize',  '~> 0.8'
+  s.add_dependency 'dotenv',    '~> 2.4'
+  s.add_dependency 'rake',      '~> 12.3'
+  s.add_dependency 'yajl-ruby', '~> 1.4'
 end

data/lib/elastic_manager.rb

@@ -1,15 +1,17 @@
 require 'dotenv/load'
 require 'date'
-require 'http'
 require 'elastic_manager/config'
 require 'elastic_manager/logger'
+require 'elastic_manager/request'
+require 'elastic_manager/utils'
 
 class ElasticManager
-
   include Config
   include Logging
+  include Request
+  include Utils
 
-  attr_reader :config
+  # attr_reader :config
 
   def initialize(argv)
     if argv.size == 0
@@ -17,87 +19,8 @@ class ElasticManager
     else
       @config = load_from_argv(argv)
     end
-  end
 
-  def true?(obj)
-    obj.to_s.downcase == 'true'
-  end
-
-  def es_request(verb, uri, json = {})
-    tries ||= @config['retry'].to_i
-
-    request_uri = "#{@config['es']['url']}#{uri}"
-
-    response = HTTP.timeout(
-                write:   @config['timeout']['write'].to_i,
-                connect: @config['timeout']['connect'].to_i,
-                read:    @config['timeout']['read'].to_i
-              ).headers(
-                'Accept':       'application/json',
-                'Content-type': 'application/json'
-              ).request(
-                verb.to_sym,
-                request_uri,
-                json: json
-              )
-
-    if response.code == 200 || response.code == 404
-      # TODO (anton.raybov): mb we need to return only one value but empty if 404???
-      return response.code, response.body.to_s
-    else
-      log.fatal "error in request - url: #{request_uri}, status: #{response.code}, response: #{response.body}"
-      exit 1
-    end
-
-  rescue StandardError => e
-
-    log.warn "try #{tries + 1} '''#{e.message}''' sleeping #{@config['sleep']} sec..."
-    sleep @config['sleep'].to_i
-
-    retry unless (tries -= 1).zero?
-    abort "backtrace:\n\t#{e.backtrace.join("\n\t")}".red
-  end
-
-  def es_green?
-    status, response = es_request('get', '/_cluster/health')
-    return JSON.parse(response)['status'] == 'green' if status == 200
-    false
-  end
-
-  def es_all_indices(from=nil, to=nil, state=nil, type=nil)
-    req_path   =  '/_cluster/state/metadata/'
-    req_params =  '?filter_path=metadata.indices.*.state,'
-    req_params << 'metadata.indices.*.settings.index.routing.allocation.require.box_type'
-
-    status, response = es_request('get', req_path + req_params)
-    if status == 200
-      indices = JSON.parse(response)['metadata']['indices']
-    else
-      log.fatal "can't work with all_indices response was: #{status} - #{response}"
-      exit 1
-    end
-
-    indices.select!{ |k, v| v['state'] == state } if state
-
-    if type
-      # TODO (anton.ryabov): next line just for debug purpose, need better handling
-      indices.each { |k, v| log.warn "#{k} - #{v.to_json}" unless v['settings'] }
-      indices.select!{ |k, v| v['settings']['index']['routing']['allocation']['require']['box_type'] == type }
-    end
-
-    res = []
-    indices.each_key do |index|
-      begin
-        index_date = Date.parse(index.gsub('-', ''))
-      rescue ArgumentError => e
-        log.error "#{e.message} for #{index}"
-        next
-      end
-
-      res << URI.escape(index) if (from..to).cover? index_date
-    end
-
-    res
+    @elastic = Request::Elastic.new(@config)
   end
 
   def work
@@ -107,104 +30,60 @@ class ElasticManager
       date_to   = Date.parse(@config['to'])
 
       unless true?(@config['force'])
-        unless es_green?
+        unless @elastic.green?
           log.fatal "elasticsearch on #{@config['es']['url']} is not green"
           exit 1
         end
       end
 
       if indices.length == 1 && indices.first == '_all'
-        indices = es_all_indices(date_from, date_to, 'close')
+        # indices = es_all_indices(date_from, date_to, 'close')
+        indices = @elastic.all_indices(date_from, date_to, 'close')
       end
 
       date_from.upto(date_to) do |date|
         date = date.to_s.tr!('-', '.')
 
         indices.each do |index_name|
-          if @config['skip']['open'].include?(index_name)
-            log.warn "#{index_name} index open skiped"
-            next
+          if @config['settings'][index_name]
+            if @config['settings'][index_name]['skip_open']
+              log.debug @config['settings'][index_name]['skip_open'].inspect
+
+              if true?(@config['settings'][index_name]['skip_open'])
+                log.warn "#{index_name} index open skiped"
+                next
+              end
+            end
           end
 
           index = "#{index_name}-#{date}"
 
-          status, response = es_request('get', "/_cat/indices/#{index}")
-          if status == 404
-            log.warn "#{index} index not found"
-            log.info "trying snapshot restore for #{index}"
-
-            snapshot_name = "snapshot_#{index}"
-
-            # TODO: we need improve this if several snapshot repos used in elastic
-            status, response = es_request('get', '/_snapshot/')
-            if status == 200
-              snapshot_repo = JSON.parse(response).keys.first
-
-              status, response = es_request('get', "/_snapshot/#{snapshot_repo}/#{snapshot_name}/")
-              if status == 200
-                snapshot = JSON.parse(response)['snapshots']
-                if snapshot.size == 1
-                  body = {
-                    index_settings: {
-                      'index.number_of_replicas'                  => 0,
-                      'index.refresh_interval'                    => -1,
-                      'index.routing.allocation.require.box_type' => 'warm'
-                    }
-                  }
-                  status, response = es_request('post', "/_snapshot/#{snapshot_repo}/#{snapshot.first['snapshot']}/_restore", body)
+          response = @elastic.request(:get, "/_cat/indices/#{index}")
 
-                  if status == 200
-                    sleep 5
-                    restore_ok = false
-                    until restore_ok
-                      sleep 30
-                      status, response = es_request('get', "/#{index}/_recovery")
+          if response.code == 404
+            log.warn "#{index} index not found"
+            log.info "#{index} trying snapshot restore"
 
-                      # TODO: add logging of percent and time ?
-                      restore_ok = JSON.parse(response)[index]['shards'].map { |s| s['stage'] == 'DONE' }.all?{ |a| a == true }
-                    end
-                    log.info "#{index} restored"
-                  else
-                    log.fatal "can't restore snapshot response was: #{status} - #{response}"
-                    exit 1
-                  end
-                else
-                  log.fatal "wrong snapshot size"
-                  exit 1
-                end
-              else
-                log.fatal "can't work with snapshot response was: #{status} - #{response}"
-                exit 1
-              end
+            if @elastic.restore_snapshot(index)
+              log.info "#{index} restored"
             else
-              log.fatal "can't work with snapshot response was: #{status} - #{response}"
-              exit 1
+              log.error "#{index} troubles with restore"
             end
-          elsif status == 200
-            if response =~ /open/
+          elsif response.code == 200
+            if response.body.to_s =~ /open/
               log.warn "#{index} index already opened"
               next
             end
 
-            begin
-              status, response = es_request('post', "/#{index}/_open?master_timeout=3m")
-              if status == 200
-                response = JSON.parse(response)
-              else
-                log.fatal "wrong response code for #{index} open"
-                exit 1
-              end
-            rescue JSON::ParserError => e
-              log.fatal "json parse err: '''#{e.message}'''\n\t#{e.backtrace.join("\n\t")}"
-              exit 1
-            end
-
-            if response['acknowledged'] == true
+            if @elastic.open_index(index)
               log.info "#{index} index open success"
             else
               log.fatal "#{index} index open failed"
               exit 1
             end
+          else
+            log.fatal "can't work with index #{index} response was: #{response.code} - #{response}"
+            exit 1
           end
         end
       end

data/lib/elastic_manager/config.rb

@@ -1,13 +1,14 @@
+require 'json'
+require 'yajl'
 require 'elastic_manager/logger'
 
 module Config
-
   include Logging
 
   MAIN_PARAMS = %w[TASK INDICES FROM TO]
   MAIN_PARAMS.freeze
 
-  ADDITIONAL_PARAMS = %w[ES_URL TIMEOUT_WRITE TIMEOUT_CONNECT TIMEOUT_READ RETRY SLEEP FORCE SKIP_OPEN]
+  ADDITIONAL_PARAMS = %w[ES_URL TIMEOUT_WRITE TIMEOUT_CONNECT TIMEOUT_READ RETRY SLEEP FORCE SETTINGS]
   ADDITIONAL_PARAMS.freeze
 
   BANNER_ENV = "Missing argument: #{MAIN_PARAMS.join(', ')}. "
@@ -28,11 +29,49 @@ module Config
     default['timeout']['write']   = '2'
     default['timeout']['connect'] = '3'
     default['timeout']['read']    = '60'
+    default['settings']           = {}
 
     log.debug "default config: #{default.inspect}"
     default
   end
 
+  def parse_settings(json)
+    begin
+      JSON.parse(json)
+    rescue JSON::ParserError => e
+      log.fatal "json parse err: '''#{e.message}'''\n\t#{e.backtrace.join("\n\t")}"
+      exit 1
+    end
+  end
+
+  def option_parser(result)
+    OptionParser.new do |parser|
+      MAIN_PARAMS.each do |param|
+        parser.on("--#{param.downcase}=#{param}") do |pr|
+          result[param.downcase] = pr
+        end
+      end
+
+      ADDITIONAL_PARAMS.each do |param|
+        parser.on("--#{param.downcase}=#{param}") do |pr|
+          params = param.split('_')
+
+          if params.length == 2
+            result[params[0].downcase][params[1].downcase] = pr
+          elsif params.length == 1
+            if params[0].downcase == 'settings'
+              result[params[0].downcase] = parse_settings(pr)
+            else
+              result[params[0].downcase] = pr
+            end
+          end
+        end
+      end
+    end.parse!
+
+    result
+  end
+
   def load_from_env
     log.debug "will load config from ENV variables"
 
@@ -54,7 +93,11 @@ module Config
         if vars.length == 2
           result[vars[0].downcase][vars[1].downcase] = ENV[var]
         elsif vars.length == 1
-          result[vars[0].downcase] = ENV[var]
+          if vars[0].downcase == 'settings'
+            result[vars[0].downcase] = parse_settings(ENV[var])
+          else
+            result[vars[0].downcase] = ENV[var]
+          end
         end
       end
     end
@@ -69,36 +112,9 @@ module Config
     log.debug "will load config from passed arguments"
     result = make_default_config
 
-    optparse = OptionParser.new do |parser|
-      MAIN_PARAMS.each do |param|
-        parser.on("--#{param.downcase}=#{param}") do |pr|
-          result[param.downcase] = pr
-        end
-      end
+    result = option_parser(result)
 
-      ADDITIONAL_PARAMS.each do |param|
-        parser.on("--#{param.downcase}=#{param}") do |pr|
-          params = param.split('_')
-
-          if vars.length == 2
-            result[params[0].downcase][params[1].downcase] = pr
-          elsif vars.length == 1
-            result[params[0].downcase] = pr
-          end
-        end
-      end
-    end
-
-    begin
-      optparse.parse!
-
-      mandatory = MAIN_PARAMS.map { |p| p.downcase }
-      if mandatory.map { |key| result[key].empty? }.any?{ |a| a == true }
-        raise OptionParser::MissingArgument.new(mandatory.join(', '))
-      end
-    rescue OptionParser::InvalidOption, OptionParser::MissingArgument
-      # puts $!.to_s
-      # puts optparse
+    if MAIN_PARAMS.map { |p| p.downcase }.map { |key| result[key].empty? }.any?{ |a| a == true }
       log.fatal BANNER_ARGV
       exit 1
     end

data/lib/elastic_manager/request.rb

@@ -0,0 +1,188 @@
+require 'http'
+require 'yajl'
+require 'elastic_manager/logger'
+require 'elastic_manager/utils'
+
+module Request
+  class Error < StandardError; end
+  class Throttling < Error; end
+  class ServerError < Error; end
+
+  class Elastic
+    include Logging
+    include Utils
+
+    RETRY_ERRORS = [StandardError, RuntimeError, Throttling]
+
+    def initialize(config)
+      @client = HTTP.timeout(
+        write:   config['timeout']['write'].to_i,
+        connect: config['timeout']['connect'].to_i,
+        read:    config['timeout']['read'].to_i
+      ).headers(
+        'Accept':       'application/json',
+        'Content-type': 'application/json'
+      )
+      @url   = config['es']['url']
+      @retry = config['retry'].to_i
+      @sleep = config['sleep'].to_i
+    end
+
+    def with_retry
+      tries ||= @retry
+
+      yield
+
+    rescue *RETRY_ERRORS => e
+      log.warn "tries left #{tries + 1} '''#{e.message}''' sleeping #{@sleep} sec..."
+      sleep @sleep
+
+      retry unless (tries -= 1).zero?
+      log.fatal "backtrace:\n\t#{e.backtrace.join("\n\t")}"
+      exit 1
+    end
+
+    def request(method, url, body={})
+      uri = @url + url
+      log.debug "uri: #{uri}"
+
+      with_retry do
+        response = @client.request(method, uri, json: body)
+
+        if response.code == 503
+          raise Request::Throttling.new(response)
+        elsif response.status.server_error?
+          raise Request::ServerError.new(response)
+        end
+
+        response
+      end
+    end
+
+    def green?
+      response = request(:get, '/_cluster/health')
+      return json_parse(response)['status'] == 'green' if response.code == 200
+      false
+    end
+
+    def all_indices(from=nil, to=nil, state=nil, type=nil)
+      indices = get_all_indices
+
+      # TODO (anton.ryabov): next line just for debug purpose, need better handling
+      indices.each { |k, v| log.debug "#{k} - #{v.to_json}" unless v['settings'] }
+
+      indices.select!{ |_, v| v['state'] == state } if state
+      indices.select!{ |_, v| v['settings']['index']['routing']['allocation']['require']['box_type'] == type } if type
+
+      indices.map do |index, _|
+        begin
+          index_date = Date.parse(index.gsub('-', ''))
+        rescue ArgumentError => e
+          log.error "#{e.message} for #{index}"
+          next
+        end
+
+        URI.escape(index) if (from..to).cover? index_date
+      end
+    end
+
+    def get_all_indices
+      req_path   =  '/_cluster/state/metadata/'
+      req_params =  '?filter_path=metadata.indices.*.state,'
+      req_params << 'metadata.indices.*.settings.index.routing.allocation.require.box_type'
+
+      response = request(:get, req_path + req_params)
+
+      if response.code == 200
+        return json_parse(response)['metadata']['indices']
+      else
+        log.fatal "can't work with all_indices response was: #{response.code} - #{response}"
+        exit 1
+      end
+    end
+
+    def find_snapshot_repo
+      # TODO: we need improve this if several snapshot repos used in elastic
+      response = request(:get, '/_snapshot/')
+
+      if response.code == 200
+        json_parse(response).keys.first
+      else
+        log.fatal "dunno what to do with: #{response.code} - #{response}"
+        exit 1
+      end
+    end
+
+    def find_snapshot(repo, snapshot_name)
+      response = request(:get, "/_snapshot/#{repo}/#{snapshot_name}/")
+
+      if response.code == 200
+        snapshot = json_parse(response)['snapshots']
+
+        if snapshot.size == 1
+          snapshot.first['snapshot']
+        else
+          log.fatal "wrong snapshot size"
+          exit 1
+        end
+      else
+        log.fatal "can't find snapshot #{snapshot_name} in #{repo} response was: #{response.code} - #{response}"
+        exit 1
+      end
+    end
+
+    def restore_snapshot(index)
+      snapshot_name = "snapshot_#{index}"
+      snapshot_repo = find_snapshot_repo
+      snapshot      = find_snapshot(snapshot_repo, snapshot_name)
+
+      body = {
+        index_settings: {
+          'index.number_of_replicas'                  => 0,
+          'index.refresh_interval'                    => -1,
+          'index.routing.allocation.require.box_type' => 'warm'
+        }
+      }
+      response = request(:post, "/_snapshot/#{snapshot_repo}/#{snapshot}/_restore", body)
+
+      if response.code == 200
+        sleep 5
+        wait_snapshot_restore(index)
+      else
+        log.fatal "can't restore snapshot #{snapshot_name} response was: #{response.code} - #{response}"
+        exit 1
+      end
+    end
+
+    def wait_snapshot_restore(index)
+      restore_ok = false
+
+      until restore_ok
+        sleep @sleep / 2
+        response = request(:get, "/#{index}/_recovery")
+
+        if response.code == 200
+          # TODO anton.ryabov: add logging of percent and time ?
+          restore_ok = json_parse(response)[index]['shards'].map { |s| s['stage'] == 'DONE' }.all?{ |a| a == true }
+        else
+          log.error "can't check recovery: #{response.code} - #{response}"
+        end
+      end
+
+      true
+    end
+
+    def open_index(index)
+      response = request(:post, "/#{index}/_open?master_timeout=1m")
+
+      if response.code == 200
+        response = json_parse(response)
+      else
+        log.fatal "wrong response code for #{index} open"
+        exit 1
+      end
+
+      return response['acknowledged'] == true
+    end
+  end
+end

data/lib/elastic_manager/utils.rb

@@ -0,0 +1,17 @@
+require 'json'
+require 'yajl'
+
+module Utils
+  include Logging
+
+  def true?(obj)
+    obj.to_s.downcase == 'true'
+  end
+
+  def json_parse(string)
+    JSON.parse(string)
+  rescue JSON::ParserError => e
+    log.fatal "json parse err: '''#{e.message}'''\n\t#{e.backtrace.join("\n\t")}"
+    exit 1
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_manager
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Antony Ryabov
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 description: Manager for logstash indices in elastic
 email: mail@doam.ru
 executables:
@@ -69,6 +97,7 @@ files:
 - lib/elastic_manager/config.rb
 - lib/elastic_manager/logger.rb
 - lib/elastic_manager/request.rb
+- lib/elastic_manager/utils.rb
 homepage: https://github.com/onetwotrip/elastic_manager
 licenses:
 - MIT
@@ -89,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Because qurator sucks