elastic_manager 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94343d3bac2671d6a49463c67d406afb79738edd347b63899ae32bcb58b1ac5e
-  data.tar.gz: aa36442960604bf810217e82d18c927db87d457e4d4c608202d85870ed190673
+  metadata.gz: d7b65a40dfcbcc2632b136bc931cdf0d1118d48281fa31ac134433ca6b2fac96
+  data.tar.gz: '0941e34bc0ef2beb51cec9d68d19e3c1c6f91060546f85a6a1838be20c173c72'
 SHA512:
-  metadata.gz: 892bf769d7e3392b4ce3dea9d3062b4ee3c2d7bb0335008478a6444d0b51c61d872b0d202ded7e978c854533b2150f08e560dacbf7dc04eb514d4bc9225358ef
-  data.tar.gz: aadad9042c0bd64848daf39c746639d3e8edcc0ad681686941ecd413324cba29375fdb1c586423abc727fef2e04cbaffe64b171f93c3cb1729640bf2763e0563
+  metadata.gz: 362ddbdfd8a8fdfdd440d0dd959ad0ba541d259501b17a0ce5ad7ede07fe0664acb7bcee6440f2c0b53c3aa868901f7f978ed620f56dab7b60abfd0252b7bebc
+  data.tar.gz: 0f50e67342101c754c7314aed5e7188fc762e2615a9946c14e36c78aa2442dce8e89634fd97d642e580aaec7d98f2f8926a0db7c58d74f6faecfb98c6370e608

data/README.md

@@ -1 +1,14 @@
+# elastic_manager
+
+Manager for logstash indices in elasticsearch. Why? Because qurator sucks!
+
 [![Build Status](https://travis-ci.org/onetwotrip/elastic_manager.svg?branch=master)](https://travis-ci.org/onetwotrip/elastic_manager)
+
+Progress:
+
+- [x] Open closed indices
+- [x] Open indices in snapshot (restore snapshot)
+- [x] Close indices
+- [ ] Chill indices
+- [ ] Snapshot indices
+- [ ] Delete snapshots

data/bin/elastic_manager

@@ -1,9 +1,10 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'elastic_manager'
 
 STDOUT.sync = true
 
-elastic_manager = ElasticManager.new(ARGV)
+elastic_manager = ElasticManager.new()
 
 elastic_manager.run

data/elastic_manager.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name        = 'elastic_manager'
   s.executables = ['elastic_manager']
-  s.version     = '0.1.4'
+  s.version     = '0.1.5'
   s.date        = '2018-10-15'
   s.summary     = 'Because qurator sucks'
   s.description = 'Manager for logstash indices in elastic'

data/lib/elastic_manager.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'dotenv/load'
 require 'date'
 require 'elastic_manager/config'
@@ -7,6 +9,7 @@ require 'elastic_manager/utils'
 require 'elastic_manager/open'
 require 'elastic_manager/close'
 
+# Main
 class ElasticManager
   include Config
   include Logging
@@ -15,17 +18,15 @@ class ElasticManager
   include Open
   include Close
 
-  def initialize(argv)
-    if argv.size == 0
-      @config = load_from_env
-    else
-      @config = load_from_argv
-    end
+  def initialize
+    @config = load_from_env
 
     @elastic = Request::Elastic.new(@config)
   end
 
   def run
-    action(@config['task'].downcase)
+    if @config['task'].casecmp('open').zero?
+      open
+    end
   end
 end

data/lib/elastic_manager/config.rb

@@ -1,23 +1,28 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'yajl'
 require 'elastic_manager/logger'
 
+# Read, validate and merge with default config
 module Config
   include Logging
 
-  MAIN_PARAMS = %w[TASK INDICES FROM TO]
-  MAIN_PARAMS.freeze
-
-  ADDITIONAL_PARAMS = %w[ES_URL TIMEOUT_WRITE TIMEOUT_CONNECT TIMEOUT_READ RETRY SLEEP FORCE SETTINGS]
-  ADDITIONAL_PARAMS.freeze
-
-  BANNER_ENV = "Missing argument: #{MAIN_PARAMS.join(', ')}. "
-  BANNER_ENV << "Usage: #{MAIN_PARAMS.map{ |p| "#{p}=#{p}"}.join(' ')} "
-  BANNER_ENV << "#{ADDITIONAL_PARAMS.map{ |p| "#{p}=#{p}"}.join(' ')} elastic_manager"
-
-  BANNER_ARGV = "Missing argument: #{MAIN_PARAMS.join(', ')}. "
-  BANNER_ARGV << "Usage: elastic_manager #{MAIN_PARAMS.map{ |p| "--#{p.downcase}=#{p.downcase}"}.join(' ')} "
-  BANNER_ARGV << "#{ADDITIONAL_PARAMS.map{ |p| "--#{p.downcase}=#{p.downcase}"}.join(' ')}"
+  PARAMS = %w[
+    TASK
+    INDICES
+    FROM
+    TO
+    DAYSAGO
+    ES_URL
+    TIMEOUT_WRITE
+    TIMEOUT_CONNECT
+    TIMEOUT_READ
+    RETRY
+    SLEEP
+    FORCE
+    SETTINGS
+  ].freeze
 
   def make_default_config
     default = Hash.new { |hash, key| hash[key] = Hash.new(&hash.default_proc) }
@@ -28,97 +33,61 @@ module Config
     default['force']              = 'false'
     default['timeout']['write']   = '2'
     default['timeout']['connect'] = '3'
-    default['timeout']['read']    = '60'
+    default['timeout']['read']    = '120'
+    default['daysago']            = ''
     default['settings']           = {}
+    default['daysago']            = ''
 
     log.debug "default config: #{default.inspect}"
     default
   end
 
-  def option_parser(result)
-    OptionParser.new do |parser|
-      MAIN_PARAMS.each do |param|
-        parser.on("--#{param.downcase}=#{param}") do |pr|
-          result[param.downcase] = pr
-        end
-      end
-
-      ADDITIONAL_PARAMS.each do |param|
-        parser.on("--#{param.downcase}=#{param}") do |pr|
-          params = param.split('_')
-
-          if params.length == 2
-            result[params[0].downcase][params[1].downcase] = pr
-          elsif params.length == 1
-            if params[0].downcase == 'settings'
-              result[params[0].downcase] = json_parse(pr)
-            else
-              result[params[0].downcase] = pr
-            end
-          end
-        end
-      end
-    end.parse!
-
-    result
+  def check_settings(var)
+    if var.casecmp('settings').zero?
+      json_parse(ENV[var])
+    else
+      ENV[var]
+    end
   end
 
-  def get_env_vars(var, result)
-    vars = var.split('_')
+  def env_parser(config)
+    PARAMS.each do |var|
+      next if ENV[var] == '' || ENV[var].nil?
 
-    if vars.length == 2
-      result[vars[0].downcase][vars[1].downcase] = ENV[var]
-    elsif vars.length == 1
-      if vars[0].downcase == 'settings'
-        result[vars[0].downcase] = json_parse(ENV[var])
-      else
-        result[vars[0].downcase] = ENV[var]
+      vars = var.split('_')
+
+      if vars.length == 2
+        config[vars[0].downcase][vars[1].downcase] = ENV[var]
+      elsif vars.length == 1
+        config[vars[0].downcase] = check_settings(vars[0])
       end
     end
 
-    result
+    config
   end
 
-  def env_parser(result)
-    MAIN_PARAMS.each do |var|
-      if ENV[var] == '' || ENV[var].nil?
-        log.fatal BANNER_ENV
-        exit 1
-      end
+  # def present?
+  #   !blank?
+  # end
 
-      result[var.downcase] = ENV[var]
+  def exit_if_invalid(config)
+    if config['task'].empty? || config['indices'].empty?
+      fail_and_exit('not enough env variables. TASK, INDICES')
     end
 
-    ADDITIONAL_PARAMS.each do |var|
-      result = get_env_vars(var, result) unless ENV[var] == '' || ENV[var].nil?
+    unless (config['from'].empty? && config['to'].empty?) || config['daysago'].empty?
+      fail_and_exit('not enough env variables. FROM/TO or DAYSAGO')
     end
-
-    result
   end
 
   def load_from_env
-    log.debug "will load config from ENV variables"
+    log.debug 'will load config from ENV variables'
 
-    result = make_default_config
-    result = env_parser(result)
-
-    log.debug "env config: #{result.inspect}"
-    result
-  end
-
-  def load_from_argv
-    require 'optparse'
-
-    log.debug "will load config from passed arguments"
-    result = make_default_config
-    result = option_parser(result)
-
-    if MAIN_PARAMS.map { |p| p.downcase }.map { |key| result[key].empty? }.any?{ |a| a == true }
-      log.fatal BANNER_ARGV
-      exit 1
-    end
+    config = make_default_config
+    config = env_parser(config)
+    exit_if_invalid(config)
 
-    log.debug "argv config: #{result.inspect}"
-    result
+    log.debug "env config: #{config.inspect}"
+    config
   end
 end

data/lib/elastic_manager/logger.rb

@@ -1,16 +1,19 @@
+# frozen_string_literal: true
+
 require 'logger'
 require 'colorize'
 
+# Universal global logging
 module Logging
 
   SEVERITY_COLORS = {
-    'DEBUG'   => 'cyan',
-    'INFO'    => 'green',
-    'WARN'    => 'yellow',
-    'ERROR'   => 'light_red',
-    'FATAL'   => 'red',
-    'UNKNOWN' => 'magenta'
-  }
+    DEBUG:   'cyan',
+    INFO:    'green',
+    WARN:    'yellow',
+    ERROR:   'light_red',
+    FATAL:   'red',
+    UNKNOWN: 'magenta'
+  }.freeze
 
   def log
     @log ||= Logging.logger_for(self.class.name)

data/lib/elastic_manager/open.rb

@@ -1,9 +1,109 @@
+# frozen_string_literal: true
+
+require 'elastic_manager/logger'
+
+# Index opening operations
 module Open
-  def do_open(indices, date)
-    indices.each do |index_name|
-      next if skip?('open', index_name)
+  include Logging
+
+  def open_prechecks(date_from, date_to)
+    unless date_from.nil?
+      if date_from > date_to
+        log.fatal "wrong dates: date to is behind date from. from: #{date_from}, to: #{date_to}"
+        exit 1
+      end
+    end
+
+    unless true?(@config['force']) && @elastic.green?
+      fail_and_exit("elasticsearch on #{@config['es']['url']} is not green")
+    end
+  end
+
+  def skip_open?(index)
+    index_name = index.split('-')[0..-2].join('-')
+
+    if @config['settings'][index_name] && @config['settings'][index_name]['skip_open']
+      if true?(@config['settings'][index_name]['skip_open'])
+        log.warn "#{index_name} index open skiped"
+        return true
+      end
+    end
+
+    false
+  end
+
+  def index_exist?(response)
+    if response.code == 200
+      true
+    elsif response.code == 404
+      false
+    else
+      log.fatal "wtf in index_exist? response was: #{response.code} - #{response}"
+      exit 1
+    end
+  end
+
+  def already_open?(response)
+    index = json_parse(response).first
+    if index['status'] == 'open'
+      log.warn "#{index['index']} index status already open"
+      return true
+    end
+
+    false
+  end
+
+  def open_prepare_vars
+    indices   = @config['indices'].split(',')
+    daysago   = @config['daysago'].to_i
+    date_from = @config['from']
+    date_to   = @config['to']
+
+    date_from = date_from.empty? ? nil : Date.parse(date_from)
+    date_to   = date_to.empty? ? nil : Date.parse(date_to)
+
+    [indices, date_from, date_to, daysago]
+  end
+
+  def action_with_log(action, index)
+    if @elastic.send(action, index)
+      log.info "#{index} #{action} succes"
+    else
+      log.error "#{index} #{action} fail"
+    end
+  end
+
+  def populate_indices(indices, date_from, date_to, daysago)
+    result = []
+
+    if indices.length == 1 && indices.first == '_all'
+      result = @elastic.all_indices(date_from, date_to, daysago, 'close')
+      result += @elastic.all_indices_in_snapshots(date_from, date_to, daysago)
+      return result
+    end
+
+    if date_from.nil?
+      result = @elastic.all_indices(date_from, date_to, daysago, 'close')
+      result += @elastic.all_indices_in_snapshots(date_from, date_to, daysago)
+      return result.select { |r| r.start_with?(*indices) }
+    else
+      date_from.upto(date_to) do |date|
+        indices.each do |index|
+          result << "#{index}-#{date.to_s.tr!('-', '.')}"
+        end
+      end
+    end
+
+    return result unless result.empty?
+
+    log.fatal 'no indices for work'
+    exit 1
+  end
+
+  def do_open(indices)
+    indices.each do |index|
+      next if skip_open?(index)
 
-      index    = "#{index_name}-#{date}"
       response = @elastic.request(:get, "/_cat/indices/#{index}")
 
       if index_exist?(response)
@@ -18,4 +118,14 @@ module Open
       end
     end
   end
+
+  def open
+    indices, date_from, date_to, daysago = open_prepare_vars
+    open_prechecks(date_from, date_to)
+    indices = populate_indices(indices, date_from, date_to, daysago)
+
+    log.debug indices.inspect
+
+    do_open(indices)
+  end
 end

data/lib/elastic_manager/request.rb

@@ -1,18 +1,23 @@
+# frozen_string_literal: true
+
 require 'http'
 require 'yajl'
 require 'elastic_manager/logger'
 require 'elastic_manager/utils'
+require 'cgi'
 
+# All kind of requests
 module Request
   class Error < StandardError; end
   class Throttling < Error; end
   class ServerError < Error; end
 
+  # Elasticsearch requests wrapper
   class Elastic
     include Logging
     include Utils
 
-    RETRY_ERRORS = [StandardError, RuntimeError, Throttling]
+    RETRY_ERRORS = [StandardError, RuntimeError, Throttling].freeze
 
     def initialize(config)
       @client = HTTP.timeout(
@@ -62,30 +67,71 @@ module Request
     def green?
       response = request(:get, '/_cluster/health')
       return json_parse(response)['status'] == 'green' if response.code == 200
+
       false
     end
 
-    def all_indices(from=nil, to=nil, state=nil, type=nil)
+    def all_indices_in_snapshots(from=nil, to=nil, daysago=nil)
+      all_snapshots = get_all_snapshots
+      all_snapshots.select! { |snap| snap['status'] == 'SUCCESS' }
+
+      result = []
+      all_snapshots.each do |snap|
+        begin
+          snap_date = Date.parse(snap['id'].gsub('-', ''))
+        rescue ArgumentError => e
+          log.error "#{e.message} for #{index}"
+          next
+        end
+
+        if from.nil? && snap_date < (Date.today - daysago)
+          result << CGI.escape(snap['id'].gsub('snapshot_', ''))
+        elsif (from..to).cover? snap_date
+          result << CGI.escape(snap['id'].gsub('snapshot_', ''))
+        end
+      end
+
+      result
+    end
+
+    def get_all_snapshots
+      snapshot_repo = find_snapshot_repo
+      response = request(:get, "/_cat/snapshots/#{snapshot_repo}")
+
+      if response.code == 200
+        json_parse(response)
+      else
+        log.fatal "can't work with all_snapshots response was: #{response.code} - #{response}"
+        exit 1
+      end
+    end
+
+    def all_indices(from=nil, to=nil, daysago=nil, state=nil, type=nil)
       indices = get_all_indices
 
-      # TODO (anton.ryabov): next line just for debug purpose, need better handling
+      # TODO: (anton.ryabov) next line just for debug purpose, need better handling
       indices.each { |k, v| log.debug "#{k} - #{v.to_json}" unless v['settings'] }
 
-      indices.select!{ |_, v| v['state'] == state } if state
-      indices.select!{ |_, v| v['settings']['index']['routing']['allocation']['require']['box_type'] == type } if type
+      indices.select! { |_, v| v['state'] == state } if state
+      indices.select! { |_, v| v['settings']['index']['routing']['allocation']['require']['box_type'] == type } if type
 
-      indices.select! do |index, _|
+      result = []
+      indices.each do |index, _|
         begin
           index_date = Date.parse(index.gsub('-', ''))
         rescue ArgumentError => e
           log.error "#{e.message} for #{index}"
           next
         end
-        (from..to).cover? index_date
+
+        if from.nil? && index_date < (Date.today - daysago)
+          result << CGI.escape(index)
+        elsif (from..to).cover? index_date
+          result << CGI.escape(index)
+        end
       end
 
-      indices = indices.keys.map { |i| i.split('-')[0..-2].join('-') }
-      return indices
+      result
     end
 
     def get_all_indices
@@ -96,7 +142,7 @@ module Request
       response = request(:get, req_path + req_params)
 
       if response.code == 200
-        return json_parse(response)['metadata']['indices']
+        json_parse(response)['metadata']['indices']
       else
         log.fatal "can't work with all_indices response was: #{response.code} - #{response}"
         exit 1
@@ -164,7 +210,7 @@ module Request
         response = request(:get, "/#{index}/_recovery")
 
         if response.code == 200
-          # TODO anton.ryabov: add logging of percent and time ?
+          # TODO: (anton.ryabov) add logging of percent and time ?
           restore_ok = json_parse(response)[index]['shards'].map { |s| s['stage'] == 'DONE' }.all?{ |a| a == true }
         else
           log.error "can't check recovery: #{response.code} - #{response}"
@@ -184,7 +230,7 @@ module Request
         exit 1
       end
 
-      return response['acknowledged'] == true
+      response['acknowledged'].true?
     end
   end
 end

data/lib/elastic_manager/utils.rb

@@ -1,16 +1,19 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'yajl'
 
+# Sharable methods
 module Utils
   include Logging
 
-  REVERT_STATE = {
+  TASK_TO_STATE = {
     'open'  => 'close',
     'close' => 'open'
   }
 
   def true?(obj)
-    obj.to_s.downcase == 'true'
+    obj.to_s.casecmp('true').zero?
   end
 
   def json_parse(string)
@@ -20,86 +23,8 @@ module Utils
     exit 1
   end
 
-  def prechecks(date_from, date_to)
-    if date_from > date_to
-      log.fatal "wrong dates: date to is behind date from. from: #{date_from}, to: #{date_to}"
-      exit 1
-    end
-
-    unless true?(@config['force'])
-      unless @elastic.green?
-        log.fatal "elasticsearch on #{@config['es']['url']} is not green"
-        exit 1
-      end
-    end
-  end
-
-  def prepare_vars
-    indices   = @config['indices'].split(',')
-    date_from = Date.parse(@config['from'])
-    date_to   = Date.parse(@config['to'])
-
-    return indices, date_from, date_to
-  end
-
-  def all_precheck(indices, date_from, date_to, state)
-    if indices.length == 1 && indices.first == '_all'
-      indices = @elastic.all_indices(date_from, date_to, state)
-    end
-
-    indices
-  end
-
-  def action_with_log(action, index)
-    if @elastic.index(action, index)
-      log.info "#{index} #{action} succes"
-    else
-      log.error "#{index} #{action} fail"
-    end
-  end
-
-  def index_exist?(response)
-    if response.code == 200
-      return true
-    elsif response.code == 404
-      return false
-    else
-      log.fatal "wtf in index_exist? response was: #{response.code} - #{response}"
-      exit 1
-    end
-  end
-
-  def already?(status, response, index)
-    if json_parse(response).first['status'] == status
-      log.warn "#{index} index status already #{status}"
-      return true
-    end
-
-    false
-  end
-
-  def skip?(status, index_name)
-    if @config['settings'][index_name]
-      if @config['settings'][index_name]['skip'][status]
-        log.debug @config['settings'][index_name]['skip'][status].inspect
-
-        if true?(@config['settings'][index_name]['skip'][status])
-          log.warn "#{index_name} index #{status} skiped"
-          return true
-        end
-      end
-    end
-
-    false
-  end
-
-  def action(task)
-    indices, date_from, date_to = prepare_vars
-    prechecks(date_from, date_to)
-    indices = all_precheck(indices, date_from, date_to, REVERT_STATE[task])
-
-    date_from.upto(date_to) do |date|
-      self.send("do_#{task}", indices, date.to_s.tr!('-', '.'))
-    end
+  def fail_and_exit(text)
+    log.fatal text
+    exit 1
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_manager
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Antony Ryabov