elastomer-cli 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea34604ccd371321f9065ebfa8cf9d6da429e171
-  data.tar.gz: 1ca391fea09cad5b4e548c1e6209dbc3672f2385
+  metadata.gz: 772a6ed6f013c952c66223437cd17fc820cacbe4
+  data.tar.gz: c8f8346f6209a636e714652792d2679f9c29debb
 SHA512:
-  metadata.gz: f19a455a6214b34ee3314636f04c71007268d4ce36e22d924694541aef9817069efbe8ff7888250da517bdeb8684f36b7d4f447a4448444a6a652f52abfe2394
-  data.tar.gz: 519d3486d37b3328c79cde37a4e1181d099d56499b945ed4f646dc073c9b90eef4ec0d4e6f70a65a957d5028f1e548c82e929d003965c3e9a7c5542e9eb92021
+  metadata.gz: 6c2ff9ada4ac6515f836f36415a61e9316bdc9785a6827f67ca42f09e6399a72659cb026b534e3b91f05c67b1eddf2a1360dcbfbf6c290a2e843fac71152fba0
+  data.tar.gz: fb7ed301f832fea0378f085945c986102886ce0fa5af4215a16fe9c3df82559a6d301d1ef57abb0331037790868af71c8b2448ae1d0bff2cab0a01cbae82a2ca

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.2 (2015-10-27)
+- New command `shard_stats`
+
 ## 0.3.1 (2014-12-18)
 - Updated output format for cluster settings
 - New command `cluster update` for updating

data/lib/elastomer/cli.rb

@@ -8,6 +8,7 @@ require 'elastomer/cli/base'
 require 'elastomer/cli/cluster'
 require 'elastomer/cli/node'
 require 'elastomer/cli/index'
+require 'elastomer/cli/shard_stats'
 require 'elastomer/cli/repository'
 require 'elastomer/cli/snapshot'
 

data/lib/elastomer/cli/application.rb

@@ -17,6 +17,11 @@ module Elastomer
       desc "snapshot SUBCOMMAND ...ARGS", "execute snapshot commands"
       subcommand "snapshot", Snapshot
 
+      desc "shard_stats", "show shard count and disk usage for all indices across all nodes"
+      def shard_stats
+        puts ShardStats.table
+      end
+
       desc "cat [COMMAND] [SCOPE] ...ARGS", "access the ES cat api"
       long_desc <<-LONGDESC
         Access the ES cat api. The cat api returns text formatted for humans.

data/lib/elastomer/cli/shard_stats.rb

@@ -0,0 +1,165 @@
+module Elastomer
+  module CLI
+    class ShardStats < Base
+      no_commands do
+        def self.table
+          self.new.table
+        end
+
+        def table
+          Terminal::Table.new do |table|
+            table << node_names.dup.unshift(nil)
+            table << generate_row("total", node_data)
+            table << :separator
+
+            index_data.keys.sort.each do |index_name|
+              table << generate_row(index_name, index_data[index_name])
+            end
+          end
+        end
+
+        def generate_row(title, data)
+          row = node_names.map do |node_name|
+
+            total         = total_disk_size_for_node(node_name)
+            data_for_node = data[node_name]
+            shards        = data_for_node["shards"]
+
+            if shards > 0
+              usage  = data_for_node["size"].to_f / total.to_f * 100.0
+              "%d : %.1f%" % [shards, usage]
+            end
+          end
+
+          row.unshift title
+          row
+        end
+
+        # Collect data for each index broken down by node. This is a Hash keyed by
+        # index name. The value for each index is another Hash that contains shard
+        # information for each node in the cluster that holds shards for the index.
+        #
+        #   {
+        #     "index1" => {
+        #       "storage1 => {
+        #         "shards" => 2,
+        #         "size"   => 1024    # size in bytes
+        #       },
+        #       "storage2 => {
+        #         "shards" => 1,
+        #         "size"   => 512     # size in bytes
+        #       }
+        #     },
+        #     "index2" => {
+        #       "storage2 => {
+        #         "shards" => 1,
+        #         "size"   => 2048
+        #       },
+        #       "storage3 => {
+        #         "shards" => 1,
+        #         "size"   => 2048
+        #       }
+        #     }
+        #     ...
+        #   }
+        #
+        # Only shards that are in the STARTED state will count towards these metrics.
+        def index_data
+          return @index_data if defined? @index_data
+
+          @index_data = Hash.new do |index_hash, index_name|
+            index_hash[index_name] = Hash.new do |node_hash, node_name|
+              node_hash[node_name] = {
+                "shards" => 0,
+                "size"   => 0
+              }
+            end
+          end
+
+          client.get("/_cat/shards", :h => "index,state,store,node", :bytes => "b").body.split("\n").each do |row|
+            index, state, size, node = row.split(%r/\s+/)
+            next unless state == "STARTED"
+
+            node_stats_for_index = @index_data[index][node]
+            node_stats_for_index["shards"] += 1
+            node_stats_for_index["size"]   += Integer(size)
+          end
+
+          @index_data
+        end
+
+        # Take the `index_data` and roll up shard counts and size at the node level.
+        # This information will be reported in the `node_data` method.
+        def shard_data_by_node
+          nodes = Hash.new do |hash, name|
+            hash[name] = {
+              "shards" => 0,
+              "size"   => 0
+            }
+          end
+
+          index_data.values.each do |hash|
+            hash.each do |name, data|
+              nodes[name]["shards"] += data["shards"]
+              nodes[name]["size"]   += data["size"]
+            end
+          end
+
+          nodes
+        end
+
+        # Collect data about the file system usage and shard counts for each node in
+        # the cluster.
+        #
+        #   {
+        #     "storage1" => {
+        #       "total"  => 4294967296,   # size in bytes
+        #       "free"   => 1073741824,   # size in bytes
+        #       "shards" => 17,
+        #       "size"   => 3221225472    # size in bytes
+        #     },
+        #     "storage2" => {
+        #       "total"  => 4294967296,
+        #       "free"   => 1073741824,
+        #       "shards" => 17,
+        #       "size"   => 3221225472
+        #     },
+        #     ...
+        #   }
+        #
+        def node_data
+          return @node_data if defined? @node_data
+
+          @node_data = {}
+          shard_data = shard_data_by_node
+
+          client.nodes.stats(:stats => "fs")["nodes"].values.each do |hash|
+            data_node = hash["attributes"]["data"]
+            next if data_node == "false" || data_node == false
+
+            name = hash["name"]
+            fs   = hash["fs"]["total"]
+
+            @node_data[name] = {
+              "total" => fs["total_in_bytes"],
+              "free"  => fs["available_in_bytes"]
+            }.merge(shard_data[name])
+          end
+
+          @node_data
+        end
+
+        # Returns the total size of the disk for the given cluster node.
+        def total_disk_size_for_node(name)
+          node_data[name]["total"]
+        end
+
+        # Returns the names of the nodes in the cluster sorted alphabetically.
+        def node_names
+          @node_names ||= node_data.keys.sort
+        end
+
+      end
+    end
+  end
+end

data/lib/elastomer/cli/version.rb

@@ -1,5 +1,5 @@
 module Elastomer
   module CLI
-    VERSION = "0.3.1"
+    VERSION = "0.3.2"
   end
 end

data/test/fixtures/vcr_cassettes/default.yml

@@ -527,7 +527,67 @@ http_interactions:
       - '121'
     body:
       encoding: UTF-8
-      string: "{\"acknowledged\":true,\"persistent\":{\"cluster.routing.allocation.enable\":\"all\",\"indices.ttl.interval\":\"70\"},\"transient\":{}}"
+      string: '{"acknowledged":true,"persistent":{"cluster.routing.allocation.enable":"all","indices.ttl.interval":"70"},"transient":{}}'
     http_version: 
   recorded_at: Thu, 18 Dec 2014 00:51:29 GMT
-recorded_with: VCR 2.9.2
+- request:
+    method: get
+    uri: http://127.0.0.1:19200/_cat/shards?bytes=b&h=index,state,store,node
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - Faraday v0.9.2
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - text/plain; charset=UTF-8
+      Content-Length:
+      - '444'
+    body:
+      encoding: UTF-8
+      string: "hi                STARTED 115 boxen \nbleh              STARTED 115
+        boxen \nblog-1            STARTED 115 boxen \nusers-1           STARTED 115
+        boxen \nfoo               STARTED 115 boxen \nshowcases-1       STARTED 115
+        boxen \naudit_log-2015-08 STARTED 115 boxen \nrepos-1           STARTED 115
+        boxen \nissues-1          STARTED 115 boxen \ncode-search-1     STARTED 115
+        boxen \ngists-1           STARTED 115 boxen \npull-requests-1   STARTED 115
+        boxen \n"
+    http_version: 
+  recorded_at: Tue, 27 Oct 2015 23:40:41 GMT
+- request:
+    method: get
+    uri: http://127.0.0.1:19200/_nodes/_all/stats/fs
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - Faraday v0.9.2
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json; charset=UTF-8
+      Content-Length:
+      - '899'
+    body:
+      encoding: UTF-8
+      string: '{"cluster_name":"elasticsearch_boxen_chrismwendt","nodes":{"ekfO1wDuQ8m1YLD-i39lhQ":{"timestamp":1445989241430,"name":"boxen","transport_address":"inet[/127.0.0.1:19300]","host":"Chriss-MacBook-Air.local","ip":["inet[/127.0.0.1:19300]","NONE"],"attributes":{"max_local_storage_nodes":"1"},"fs":{"timestamp":1445989241430,"total":{"total_in_bytes":499071844352,"free_in_bytes":407380561920,"available_in_bytes":407118417920,"disk_reads":0,"disk_writes":0,"disk_io_op":0,"disk_read_size_in_bytes":0,"disk_write_size_in_bytes":0,"disk_io_size_in_bytes":0},"data":[{"path":"/opt/boxen/data/elasticsearch/elasticsearch_boxen_chrismwendt/nodes/0","mount":"/","dev":"/dev/disk1","total_in_bytes":499071844352,"free_in_bytes":407380561920,"available_in_bytes":407118417920,"disk_reads":0,"disk_writes":0,"disk_io_op":0,"disk_read_size_in_bytes":0,"disk_write_size_in_bytes":0,"disk_io_size_in_bytes":0}]}}}}'
+    http_version: 
+  recorded_at: Tue, 27 Oct 2015 23:40:41 GMT
+recorded_with: VCR 2.9.3

data/test/shard_stats_test.rb

@@ -0,0 +1,13 @@
+require File.expand_path('../test_helper', __FILE__)
+
+describe Elastomer::CLI::ShardStats do
+  describe 'shard_stats' do
+    it 'should show shard stats' do
+      VCR.use_cassette('default') do
+        elastomer 'shard_stats'
+        @out.must_match /total.*12/m
+        @out.must_match /audit_log/m
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastomer-cli
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Grant Rodgers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-18 00:00:00.000000000 Z
+date: 2015-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -146,6 +146,7 @@ files:
 - lib/elastomer/cli/index.rb
 - lib/elastomer/cli/node.rb
 - lib/elastomer/cli/repository.rb
+- lib/elastomer/cli/shard_stats.rb
 - lib/elastomer/cli/snapshot.rb
 - lib/elastomer/cli/version.rb
 - script/bootstrap
@@ -165,6 +166,7 @@ files:
 - test/index_test.rb
 - test/node_test.rb
 - test/repository_test.rb
+- test/shard_stats_test.rb
 - test/snapshot_test.rb
 - test/test_helper.rb
 homepage: https://github.com/github/elastomer-cli
@@ -187,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.3
 signing_key: 
 specification_version: 4
 summary: Command line interface for elastomer
@@ -205,5 +207,7 @@ test_files:
 - test/index_test.rb
 - test/node_test.rb
 - test/repository_test.rb
+- test/shard_stats_test.rb
 - test/snapshot_test.rb
 - test/test_helper.rb
+has_rdoc: