esteem 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 340aa00a323b11ea872b48a404e951df6eabb2eb
+  data.tar.gz: e9b5ee78c4c9f76e0b041617f7984f88f884526b
+SHA512:
+  metadata.gz: 8ff6c19a502f907dd2a30d0391042e13f8bd1b646ed19d8a634a54044e2d3db6c22bb89cb820cdbf57fe1f7b233bd8b83b3b52b91cf4537475d46fcec7daf832
+  data.tar.gz: 3cc3b6cf92f7c058a9724b8b23c61f772c087bd38dceb330e9545e82732a02ced2b13ce46d8030c3a4a2e73b79346e80700859fbc54b56be7d2e4f982923f71e

data/.gitignore

@@ -0,0 +1,12 @@
+*~
+*.gem
+*.log
+*.out
+*.pid
+*.swp
+.DS_Store
+.yardoc
+doc
+pkg
+tmp
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,15 @@
+source 'http://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'rake'
+  gem 'yard'
+  gem 'version'
+  gem 'rubygems-tasks'
+end
+
+group :test do
+  gem 'minitest'
+end

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2015 Sean Clemmer
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+require 'rake'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test*.rb']
+  test.verbose = true
+end
+
+task default: :test
+
+
+require 'yard'
+YARD::Rake::YardocTask.new do |t|
+  t.files = %w[ --readme Readme.md lib/**/*.rb - VERSION ]
+end
+
+
+require 'rubygems/tasks'
+Gem::Tasks.new push: true, sign: {} do |tasks|
+  tasks.console.command = 'pry'
+end
+Gem::Tasks::Sign::Checksum.new sha2: true
+
+
+require 'rake/version_task'
+Rake::VersionTask.new

data/Readme.md

@@ -0,0 +1,3 @@
+# Esteem
+
+Tips, tricks and scripts for managing time-based Elasticsearch indexes.

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/esteem

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'esteem'
+rc = Esteem::Main.start ARGV
+exit rc unless rc.nil?

data/esteem.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path(File.join('..', 'lib'), __FILE__)
+require 'esteem/metadata'
+
+Gem::Specification.new do |s|
+  s.name        = 'esteem'
+  s.version     = Esteem::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.author      = Esteem::AUTHOR
+  s.license     = Esteem::LICENSE
+  s.homepage    = Esteem::HOMEPAGE
+  s.email       = Esteem::EMAIL
+  s.summary     = Esteem::SUMMARY
+  s.description = Esteem::SUMMARY + '.'
+
+  s.add_runtime_dependency 'thor', '~> 0'
+  s.add_runtime_dependency 'slog', '~> 1.1'
+  s.add_runtime_dependency 'oj', '~> 2.11'
+  s.add_runtime_dependency 'rest-client', '~> 1.7'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File::basename(f) }
+  s.require_paths = %w[ lib ]
+end

data/lib/esteem/helpers.rb

@@ -0,0 +1,28 @@
+require 'thor/util'
+require 'thor/actions'
+
+
+module Esteem
+
+  # Mixins for Esteem's Thor subclasses.
+  module Helpers
+
+    # Save the canonical implementation of "puts"
+    alias_method :old_puts, :puts
+
+    # Monkeypatch puts to support Thor::Shell::Color.
+    def puts *args
+      return old_puts if args.empty?
+      old_puts shell.set_color(*args)
+    end
+
+
+    # Shortcut for Thor::Util.
+    def util ; Thor::Util end
+
+
+    # Shortcut for Thor::Actions.
+    def actions ; Thor::Actions end
+
+  end
+end

data/lib/esteem/main.rb

@@ -0,0 +1,47 @@
+require 'thor'
+require 'slog'
+
+require_relative 'reindex'
+require_relative 'helpers'
+require_relative 'metadata'
+
+
+module Esteem
+
+  # Esteem's entrypoint.
+  class Main < Thor
+    include Esteem::Helpers
+
+
+    desc 'version', 'Echo the application version'
+    def version
+      puts VERSION
+      return nil
+    end
+
+
+    desc 'art', 'View the application art'
+    def art
+      puts
+      puts ART
+      puts
+      return nil
+    end
+
+
+    desc 'reindex', 'Move data to a new index gracefully'
+    option :update, type: :boolean, aliases: %w[ -u ]
+    option :frame, type: :numeric, aliases: %w[ -f ], default: 1000
+    option :source, type: :string, aliases: %w[ -s ], required: true
+    option :destination, type: :string, aliases: %w[ -d ], required: true
+    def reindex
+      reindex = Reindex.new \
+        update: options.update?,
+        frame: options.frame,
+        src: options.source,
+        dst: options.destination
+      return reindex.success
+    end
+
+  end
+end

data/lib/esteem/metadata.rb

@@ -0,0 +1,30 @@
+# Esteem gives you confidence!
+module Esteem
+
+  # Pull the project version out of the VERSION file
+  VERSION = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
+
+  # A quick summary for use in the command-line interface
+  SUMMARY = 'Tips, tricks and scripts for managing time-based Elasticsearch indexes'
+
+  # Take credit for your work
+  AUTHOR   = 'Sean Clemmer'
+
+  # Like the MIT license, but even simpler
+  LICENSE  = 'ISC'
+
+  # Where you should look first
+  HOMEPAGE = 'https://github.com/sczizzo/esteem'
+
+  # Take responsibility for your work
+  EMAIL    = 'EMAIL'
+
+  # Every project deserves its own ASCII art
+  ART      = <<-'EOART'
+     ____                                        ____         ____
+    |                        ..'''' `````|````` |            |                  .'. .`.
+    |______               .''            |      |______      |______          .'   `   `.
+    |                  ..'               |      |            |              .'           `.
+    |___________ ....''                  |      |___________ |___________ .'               `.
+  EOART
+end

data/lib/esteem/reindex.rb

@@ -0,0 +1,151 @@
+require 'oj'
+require 'rest-client'
+
+
+Oj.default_options = { mode: :compat }
+
+module Esteem
+
+  # Cribbed from geronime's "es-reindex" script: https://github.com/geronime/es-reindex
+  class Reindex
+    attr_reader :success
+
+
+    # Perform a new reindex job. Success or failure will be reported by the
+    # read-only "success" attribute.
+    #
+    # @option [String] :source URL for source index and type
+    # @option [String] :destination URL for destination index
+    # @option [Integer] :frame Scroll window size
+    # @option [Boolean] :update Set to update existing documents
+    def initialize source:, destination:, frame:, update:
+      src, dst = source, destination
+      @success = false
+
+      type, src = src.reverse.split('/', 2).map(&:reverse)
+
+      surl, durl, sidx, didx = '', '', '', ''
+      [[src, surl, sidx], [dst, durl, didx]].each{|param, url, idx|
+        if param =~ %r{^(.*)/(.*?)$}
+          url.replace $1
+          idx.replace $2
+        else
+          url.replace 'http://127.0.0.1:9200'
+          idx.replace param
+        end
+      }
+
+      puts "Copying '%s/%s/%s' to '%s/%s'" % [ surl, sidx, type, durl, didx ]
+
+      scount = retried_request :get, "#{surl}/#{sidx}/#{type}/_count?q=*"
+      scount = Oj.load(scount)['count'].to_i
+
+      puts "%d documents" % scount
+
+      if scount == 0
+        puts "Hey, this index is empty! Skipping..."
+        @success = true
+        return
+      end
+
+      t, done = Time.now, 0
+      shards = retried_request :get, "#{surl}/#{sidx}/_count?q=*"
+      shards = Oj.load(shards)['_shards']['total'].to_i
+      scan = retried_request(:get, "#{surl}/#{sidx}/#{type}/_search?search_type=scan&scroll=10m&size=#{frame / shards}")
+      scan = Oj.load scan
+      scroll_id = scan['_scroll_id']
+      total = scan['hits']['total']
+      printf "    %u/%u (%.1f%%) done.\r", done, total, 0
+
+      bulk_op = update ? "index" : "create"
+
+      loop do
+        data = retried_request(:get, "#{surl}/_search/scroll?scroll=10m&scroll_id=#{scroll_id}")
+        data = Oj.load data
+        break if data['hits']['hits'].empty?
+
+        scroll_id = data['_scroll_id']
+        bulk = ''
+
+        data['hits']['hits'].each do |doc|
+          bulk << %Q({"#{bulk_op}": {"_index" : "#{didx}", "_id" : "#{
+              doc['_id']}", "_type" : "#{doc['_type']}"}}\n)
+          bulk << Oj.dump(doc['_source']) + "\n"
+          done += 1
+        end
+
+        unless bulk.empty?
+          bulk << "\n" # empty line in the end required
+          retried_request :post, "#{durl}/_bulk", bulk
+        end
+
+        eta = total * (Time.now - t) / done
+        printf "    %u/%u (%.1f%%) done in %s, E.T.A.: %s.\r",
+          done, total, 100.0 * done / total, tm_len(Time.now - t), t + eta
+      end
+
+      printf "#{' ' * 80}\r    %u/%u done in %s.\n",
+        done, total, tm_len(Time.now - t)
+
+
+
+      tries = 0
+      dcount = -1
+
+      puts "Waiting for documents to appear in index..."
+      until total <= dcount
+        dcount = retried_request :get, "#{durl}/#{didx}/#{type}/_count"
+        dcount = Oj.load(dcount)['count'].to_i
+        tries += 1
+
+        if tries > 30
+          puts "Too many tries!"
+          @success = false
+          return
+        end
+
+        sleep 15
+      end
+
+      if done != dcount
+        puts "Incomplete done! %d != %d" % [ done, dcount ]
+        @success = false
+        return
+      end
+
+      @success = true
+    end
+
+
+
+  private
+
+    def tm_len l
+      t = []
+      t.push l/86400; l %= 86400
+      t.push l/3600;  l %= 3600
+      t.push l/60;    l %= 60
+      t.push l
+      out = sprintf '%u', t.shift
+      out = out == '0' ? '' : out + ' days, '
+      out += sprintf('%u:%02u:%02u', *t)
+      out
+    end
+
+
+    def retried_request method, url, data=nil
+      while true
+        begin
+          return data ?
+            RestClient.send(method, url, data) :
+            RestClient.send(method, url)
+        rescue RestClient::ResourceNotFound # no point to retry
+          return nil
+        rescue => e
+          warn "\nRetrying #{method.to_s.upcase} ERROR: #{e.class} - #{e.message}"
+        end
+      end
+    end
+
+  end
+end

data/lib/esteem.rb

@@ -0,0 +1 @@
+require_relative 'esteem/main'

data/test/test_esteem.rb

@@ -0,0 +1,27 @@
+require 'tmpdir'
+require 'logger'
+require 'tempfile'
+require 'fileutils'
+require 'pathname'
+
+require 'minitest/autorun'
+
+require_relative '../lib/esteem'
+
+
+
+class TestEsteem < MiniTest::Test
+  def setup
+    @tmpdir       = Dir.mktmpdir
+    @logger       = Logger.new STDERR
+    @logger.level = Logger::WARN
+  end
+
+  def teardown
+    FileUtils.rm_rf @tmpdir
+  end
+
+  def test_something
+    assert true
+  end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: esteem
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Sean Clemmer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.11'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: Tips, tricks and scripts for managing time-based Elasticsearch indexes.
+email: EMAIL
+executables:
+- esteem
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- Rakefile
+- Readme.md
+- VERSION
+- bin/esteem
+- esteem.gemspec
+- lib/esteem.rb
+- lib/esteem/helpers.rb
+- lib/esteem/main.rb
+- lib/esteem/metadata.rb
+- lib/esteem/reindex.rb
+- test/test_esteem.rb
+homepage: https://github.com/sczizzo/esteem
+licenses:
+- ISC
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Tips, tricks and scripts for managing time-based Elasticsearch indexes
+test_files:
+- test/test_esteem.rb