elasticsearch-data-cleaner 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a863f431274d88db9cbc0579f597229674981a8
+  data.tar.gz: 6fd30ed2b2c8836a7df9b83bc12c9dd811f258d5
+SHA512:
+  metadata.gz: ff3b0b5e9ec742004b1eec12cb81755253625a969d8b54e4a18a1007e684cf7b5c424ecfe0ff80448da5fdda06f560a386e83cbdeff38d681f420ec34be82f03
+  data.tar.gz: ebc636008e9177583833e6ca7d6e8e46ae241e62232beb11289ec4c7f21c2892fb13c96b8738062049684c1156fa56f64ba4b57c94b551d57be8a610790b84f9

data/bin/elasticsearch-data-cleaner

@@ -0,0 +1,157 @@
+#!/usr/bin/env ruby
+###########################################################
+###                                                     ###
+### Author: Maksim Podlesnyi <mpodlesnyi@smartling.com> ###
+###                                                     ###
+###########################################################
+
+require 'elasticsearch-data-cleaner'
+require 'optparse'
+require 'ostruct'
+require 'yaml'
+require 'logger'
+
+::CONFIGFILE_HELP = '---
+events-%Y%m%d:       # index pattern compatible ruby date format directives
+  number: 7          # how many indices script have to keep by pattern above
+  future: true       # leave indices with future timestamp. true by default
+logstash-%Y%m%d:
+  number: 60
+  future: false
+  types:             # types description
+    elb:             # type name. there is no patterns
+      number: 30
+      optimize: true # run _optimize for index after removing this type.
+                     # false by default
+    nginx-pinch-main:
+      number: 10
+      optimize: true
+    nginx-wlb-main:
+      number: 60
+      optimize: true
+
+# Attention: you have to use number of indices greater or equal numbers
+# of indices with types because first of all script deletes indices
+# using index setting.'
+
+class MyOptparse
+
+  def parse(args)
+    options = OpenStruct.new
+    options.dry_run = false
+    options.timeout = 300
+    options.url = 'http://localhost:9200'
+    options.debug = false
+    options.empty = true
+
+    @opt_parser = OptionParser.new do |opts|
+      banner = [
+        $0 + ' --dry-run --config data_rotate.yaml',
+        "\t\t\tdry run for config\n",
+        $0 + ' --debug --config data_rotate.yaml',
+        "\t\t\tenable debug mode\n",
+        $0 + ' -t 600 --config data_rotate.yaml',
+        "\t\t\tincrease http client timeout to 10 minutes\n",
+      ]
+      opts.banner = "Examples:\n\t" + banner.join("\n\t")
+
+      opts.separator ""
+      opts.separator "Specific options:"
+
+      opts.on("-u", "--url URL",
+              "Url of ES API. Default it #{options.url}") do |url|
+          options.url = url
+      end
+
+      opts.on("-t", "--timeout SECONDS",
+              "The timeout for connection to ES API. Default is #{options.timeout}") do |timeout|
+          options.timeout = timeout.to_i
+      end
+
+      opts.on("-c", "--config CONFIG",
+              "YAML file with configuration for rotate data") do |config|
+          options.config = config
+      end
+
+      opts.on("-l", "--logfile LOGFILE",
+              "Log file") do |logfile|
+          options.logfile = logfile
+      end
+
+      opts.on("--dry-run",
+              "Run script without applying any changes") do |dry_run|
+          options.dry_run = true
+      end
+
+      opts.on("--empty",
+              "Delete any empty indices") do |empty|
+          options.empty = false
+      end
+
+      opts.on("--debug",
+              'Debug mode. More verbose') do |debug|
+          options.debug = true
+      end
+
+      opts.separator "Common options:"
+
+      opts.on_tail("-h", "--help", "Show this message") do
+        puts opts
+        puts "\n\n\nConfig file (YAML) example:"
+        puts ::CONFIGFILE_HELP
+        exit(0)
+      end
+
+    end
+      @opt_parser.parse!(args)
+      options
+      rescue OptionParser::ParseError
+        $stderr.print "Error: " + $!.to_s + "\n"
+        puts @opt_parser
+        exit(-1)
+  end
+  def help
+    puts @opt_parser
+  end
+end
+
+begin
+  parser = MyOptparse.new
+  options = parser.parse(ARGV)
+  $logger = Logger.new(options.logfile ? options.logfile : STDOUT)
+  if options.debug
+    $logger.level = Logger::DEBUG
+  else
+    $logger.level = Logger::INFO
+  end
+  if options.logfile
+    $logger.formatter = proc do |severity, datetime, progname, msg|
+       "[#{datetime}] #{severity} : #{msg}\n"
+    end
+    $logger.info("begin")
+  else
+    $logger.formatter = proc do |severity, datetime, progname, msg|
+       "#{msg}\n"
+    end
+  end
+  if options.config
+    if !File.exists?(options.config)
+      raise "can not read config file #{options.config}"
+    end
+  else
+    raise 'please specify path to yaml config file'
+  end
+  config = YAML.load(IO.read(options.config))
+  ess = EScleaner.new options
+  ess.run(config)
+  if options.logfile
+    $logger.info("end")
+  end
+rescue Exception => e
+  if defined?($logger) == 'global-variable'
+    $logger.error e.message
+  else
+    $stderr.puts e.message
+  end
+  exit(1)
+end

data/lib/elasticsearch-data-cleaner.rb

@@ -0,0 +1,165 @@
+###########################################################
+###                                                     ###
+### Author: Maksim Podlesnyi <mpodlesnyi@smartling.com> ###
+###                                                     ###
+###########################################################
+require 'rubygems'
+require 'json'
+require 'faraday'
+require 'date'
+
+
+class EScleaner
+
+  def initialize(options)
+    @connection = ::Faraday.new options.url, { :request => { :timeout => options.timeout } }
+    @options = options
+    info()
+  end
+
+  def info
+    #get ES information
+    resp = @connection.get
+    @es_info = parse_response resp.body
+    @es_version = {}
+    @es_version['major'], @es_version['minor'], @es_version['patch'] = @es_info['version']['number'].split('.').map { |i| i.to_i }
+    $logger.debug("detected Elasticsearch #{@es_version['major']}.#{@es_version['minor']}")
+  end
+
+  def parse_response(response)
+    j = ::JSON.parse response
+    if j.has_key?('error')
+      raise j['error']
+    else
+      j
+    end
+  end
+
+  def acknowledged_test(response, answer)
+    # Testing ES response for errors
+    r = parse_response response.body
+    if r.has_key? answer and r[answer] == true
+    else
+      raise 'request failed'
+    end
+  end
+
+  def list
+    # Get list of indices
+    resp = @connection.get "_aliases"
+    r = parse_response resp.body
+    r.keys
+  end
+
+  def exists(index, type)
+    #checks if type exists
+    if @es_version['major'] >= 5
+      url = "#{index}/_mapping/#{type}"
+    else
+      url = "#{index}/#{type}"
+    end
+    resp = @connection.head url
+    if resp.status == 200
+      return true
+    elsif resp.status == 404
+      return false
+    else
+      raise "could not check if type #{index}/#{type} exists. got response code #{resp.status}"
+    end
+  end
+
+  def docs(index)
+    # Returns number of documents for index
+    resp = @connection.get "#{index}/_stats/docs"
+    r = (parse_response resp.body)['_all']['primaries']['docs']['count']
+    if r
+      if r < 10
+        $logger.warn("index #{index} has #{r} docs")
+      end
+      return r
+    else
+      raise "could not get count of docs for index #{index}"
+    end
+  end
+
+  def delete(object)
+    # Deleting indices or types
+    $logger.info("deleting #{object}#{@options.dry_run ? ' (dry_run)': ''}")
+    if !@options.dry_run
+      resp = @connection.delete "#{object}"
+      acknowledged_test resp, 'acknowledged'
+    end
+  end
+
+  def optimize(index)
+    # Run _optimize
+    $logger.info("starting optimize for index #{index}#{@options.dry_run ? ' (dry_run)': ''}")
+    if !@options.dry_run
+      resp = @connection.post "#{index}/_optimize?only_expunge_deletes=true"
+      $logger.debug(parse_response resp.body)
+    end
+  end
+
+  def run(config)
+    # Run rotate of ES data
+    config.each do |pattern, settings|
+      a = []
+      list().each do |index|
+        # check if empty just delete it
+        if !@options.empty and docs(index) == 0
+          delete(index)
+        else
+          begin
+            a << DateTime.strptime(index, pattern)
+          rescue
+          end
+        end
+      end
+      today = Date.today
+      sorted = a.sort
+      while sorted.length > 0
+        need_to_optimize = false
+        date = sorted.pop
+        d = date.to_date
+        index = date.strftime(pattern)
+        if d > today
+          if settings['future'] == false
+            $logger.debug("going to delete index #{index}. It is future")
+            delete(index)
+          end
+        else
+          if settings['number']
+            if settings['number'] <= 0
+              delete(index)
+            else
+              if settings['types']
+                # Checking types settings
+                settings['types'].each do |type, type_settings|
+                  if exists(index, type)
+                    if type_settings['number']
+                      if type_settings['number'] <= 0
+                        delete("#{index}/#{type}")
+                        if type_settings['optimize'] and need_to_optimize == false
+                          need_to_optimize = true
+                        end
+                      else
+                        type_settings['number'] -= 1
+                      end
+                    end
+                  end
+                end
+              end
+            end
+            settings['number'] -= 1
+          end
+        end
+        a.delete(date)
+        # do optimize if needed
+        if need_to_optimize
+          optimize(index)
+        end
+      end
+    end
+  end
+end
+

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: elasticsearch-data-cleaner
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Maksim Podlesnyi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: Command line tool which helps remove old ES indices and types
+email:
+- mpodlesnyi@smartling.com
+- itops@smartling.com
+executables:
+- elasticsearch-data-cleaner
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/elasticsearch-data-cleaner
+- lib/elasticsearch-data-cleaner.rb
+homepage: https://github.com/Smartling/elasticsearch-data-cleaner
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Command line tool for removing old Elasticsearch data (indices and types)
+test_files: []