stash-query 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ccad0b0e0077c1bbda3390027e84b57be31235ab
+  data.tar.gz: eb48df98b461d986d2f8838a10562c8c7c99597d
+SHA512:
+  metadata.gz: 11a8394d3446915a8081f70af7db9b371294e23a241d4c292e572c604347617064e70f8ea4ec924fba53f6aaac95e3aff1557b45b49d76bfda57a7632bcf4e49
+  data.tar.gz: 18b4f50a7848a14068437b80785f5e8d04fc3afc935294a403a76f8689cfb9086823ea27e199b395640c419f4ead7aa2a454a8bb3ed4f86220424dd335c7a55c

data/.gitignore

@@ -0,0 +1,3 @@
+*.lock
+*.swp
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in barcode.gemspec
+gemspec

data/README.md

@@ -0,0 +1,24 @@
+stash-query
+===========
+
+A CLI Tool for Querying Logstash and Exporting the results. Uses the Lucene query syntax that Kibana utilizes, but provides the option for exporting. 
+
+Usage:
+```
+    -c, --connect_host [HOST]        Logstash host to run query on (defaults to: localhost)
+    -p, --port [PORT]                Logstash port (defaults to: 9200)
+    -i, --index-prefix [PREFIX]      Index name prefix. Defaults to 'logstash-'
+    -w, --write [FILE]               Write output file location (defaults to nil)
+    -d, --debug                      Debug mode
+    -s, --start [DATE]               Start date. Format: YYYY-MM-DDThh:mm:ss.SSSZ. Ex: 2013-12-01T12:00:00.000Z
+    -e, --end [DATE]                 End date. Format: YYYY-MM-DDThh:mm:ss.SSSZ
+    -q, --query [QUERY]              Query string
+    -t, --tags [TAGS]                Tags to query. Comma delimited
+    -f, --write-fields [FIELDS]      Comma delimited list of Logstash fields to write to output file. Defaults to "message"
+    -l, --delimiter [DELIMITER]      Delimiter to use in output file. Defaults to ","
+```
+
+Examples:
+```
+stash-query -s 2013-12-01T00:00:00.000Z -e 2013-12-02T00:00:00.000Z -t my_tag -q 'message:hello_world' -w /tmp/my_query.txt
+```

data/bin/stash-query

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'rubygems' if RUBY_VERSION < '1.9.0'
+require 'stash-query'
+
+############ CONFIG ###########
+$config = {}
+$config[:host] = "localhost"
+$config[:port] = "9200"
+$config[:index_prefix] = [ "logstash-" ]
+$config[:scroll_size] = 10  ## Number of hits returned per scroll request. Not sure what to use here...
+$config[:scroll_time] = '30m'   
+$config[:report] = nil
+$config[:write_fields] = []
+$config[:delim] = ','
+$debug = nil
+$flush_buffer = 1000 ## Number of log lines to flush to file at
+$new_transport = true
+
+##################################
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: "
+  opts.on('-c','--connect_host [HOST]', "Logstash host to run query on (defaults to: #{$config[:host]})") { |v| $config[:host] = v unless v.empty? or v.nil? }
+  opts.on('-p','--port [PORT]', "Logstash port (defaults to: #{$config[:port]})") { |v| $config[:port] = v unless v.empty? or v.nil? }
+  opts.on('-i','--index-prefix [PREFIX]', "Index name prefix(es). Defaults to 'logstash-'. Comma delimited") do |prefix|
+    $config[:index_prefix] = prefix.split(',') unless prefix.empty? or prefix.nil?
+  end
+  opts.on('-w', '--write [FILE]', 'Write output file location (defaults to nil)') { |v| $config[:output] = v }
+  opts.on('-d', '--debug', 'Debug mode') { |v| $debug = true }
+
+  opts.on('-s', '--start [DATE]', 'Start date. Format: YYYY-MM-DDThh:mm:ss.SSSZ. Ex: 2013-12-01T12:00:00.000Z') do |v| 
+    $config[:start] = v
+  end
+
+  opts.on('-e', '--end [DATE]', 'End date. Format: YYYY-MM-DDThh:mm:ss.SSSZ') do |v| 
+    $config[:end] = v
+  end
+
+  opts.on('-q', '--query [QUERY]', 'Query string') { |v| $config[:query] = "#{v}" unless v.empty? }
+  opts.on('-t', '--tags [TAGS]', 'Tags to query. Comma delimited')  do |tags|  
+    arr = tags.split(',')
+    if arr.length > 1
+      $config[:tags] = "tags:(" + arr.join(' AND ') + ")"
+    else
+      $config[:tags] = "tags:#{tags}"
+    end
+  end
+  opts.on('-f', '--write-fields [FIELDS]', 'Comma delimited list of Logstash fields to write to output file. Defaults to "message"') do |fields|
+   $config[:write_fields] = fields.split(',')
+  end
+  opts.on('-l', '--delimiter [DELIMITER]', 'Delimiter to use in output file. Defaults to ","') do |v|
+    $config[:delim] = v
+  end
+  opts.parse!
+end
+
+es_conf = {
+  :query => $config[:query],
+  :tags => $config[:tags],
+  :start_date => $config[:start],
+  :end_date => $config[:end],
+  :output_file => $config[:output],
+  :host => $config[:host],
+  :port => $config[:port],
+  :index_prefixes => $config[:index_prefix],
+  :write_fields => $config[:write_fields],
+  :delimiter => $config[:delim],
+  :do_print => true
+}
+es = Stashquery::Query.new(es_conf)
+
+exit if es.num_results < 1
+
+until es.query_finished do
+  sleep 2
+end

data/lib/stash-query.rb

@@ -0,0 +1,3 @@
+$: << File.expand_path(File.dirname(__FILE__))
+require 'stash-query/version'
+require 'stash-query/query'

data/lib/stash-query/query.rb

@@ -0,0 +1,272 @@
+#!/usr/bin/env ruby
+
+require 'rubygems' if RUBY_VERSION < '1.9.0'
+require 'elasticsearch'
+require 'json'
+require 'date'
+require 'optparse'
+require 'curb'
+require 'progress_bar'
+
+module Stashquery
+  class Query
+
+  DEFAULT_FIELD = "message"
+  $debug = nil
+  $flush_buffer = 1000 ## Number of log lines to flush to file at
+  $new_transport = true
+
+  attr_reader :query_finished
+  attr_reader :num_results
+  attr_reader :start_date
+  attr_reader :end_date
+
+  def initialize(conf = {})
+    @config = {}
+    @config[:host] = conf[:host] || "ls2-es-lb.int.tropo.com"
+    @config[:port] = conf[:port] || "9200"
+    if conf[:index_prefixes].is_a? Array and ! conf[:index_prefixes].empty?
+      @config[:index_prefixes] = conf[:index_prefixes]
+    else
+      @config[:index_prefixes] = [ "logstash-" ]
+    end
+    @config[:scroll_size] = conf[:scroll_size] || "100"
+    @config[:scroll_time] = conf[:scroll_time] || "30m"
+    @config[:output] = conf[:output_file] || nil
+    @query = conf[:query] || nil
+    @tags = conf[:tags] || nil
+    @start_date = conf[:start_date]
+    @end_date = conf[:end_date]
+    @config[:write_fields] = []
+    set_write_fields(conf[:write_fields])
+    @config[:delimiter] = conf[:delimiter] || ','
+    @num_results = 0
+    @query_finished = false
+    @scroll_ids = Array.new
+
+    if conf[:do_print]
+      @config[:print] = true
+      require 'progress_bar'
+    end
+
+    ## Do this better
+    unless Query.validate_date(@start_date) and Query.validate_date(@end_date)
+      raise "Improper date format entered"
+    end
+
+    ## Cleanup output file. Probably a better way to do this.
+    unless @config[:output].nil?
+      begin
+        File.truncate(@config[:output],0)
+      rescue
+      end
+    end
+
+    @es_conn = connect_to_es
+    run_query
+    sort_file
+  end
+
+  def self.validate_date(str)
+    return true if str =~ /20[0-9]{2}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])T[012][0-9]:[0-5][0-9]:[0-5][0-9]\.[0-9]{3}Z/
+    return nil
+  end
+
+  private
+
+  def sort_file
+    unless @config[:output].nil?
+      arr = File.readlines(@config[:output]).sort
+      File.open(@config[:output], 'w') do |f|
+        f.puts arr
+      end
+    end
+  end
+
+  def flush_to_file(hit_list)
+    return if @config[:output].nil?
+    File.open(@config[:output], 'a') do |file|
+      begin
+        file.puts(generate_output(hit_list))
+      rescue => e
+        puts "Error writing to file."
+        raise e
+        exit
+      end
+    end
+  end
+
+  def set_write_fields(fields)
+    if fields.is_a? Array
+      if fields.empty?
+        @config[:write_fields] << DEFAULT_FIELD
+      else
+        @config[:write_fields] = fields
+      end
+    elsif fields.is_a? String
+      @config[:write_fields] = [ fields ]
+    else
+      @config[:write_fields] = [ DEFAULT_FIELD ]
+    end
+  end
+
+  def generate_output(hit_list)
+    output_data = []
+    hit_list.each do |event|
+      event_data = []
+      if @config[:write_fields].include?('_all')
+        event['_source'].keys.each do |field|
+          event_data << "#{event['_source'][field]}".gsub("\n", '')
+        end
+      else
+        @config[:write_fields].each do |field|
+          event_data << "#{event['_source'][field] if event['_source'][field]}".gsub("\n", '')
+        end
+      end
+      output_data << event_data.join(@config[:delimiter])
+    end
+    output_data
+  end
+
+  def connect_to_es
+    ## Try a different transporter
+    if $new_transport
+      require 'typhoeus'
+      require 'typhoeus/adapters/faraday'
+
+      transport_conf = lambda do |f|
+        #f.response :logger
+        f.adapter :typhoeus
+      end
+    end
+
+    ## Connect to ES server
+    begin
+      if $new_transport
+        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new hosts: [ { host: @config[:host], port: @config[:port] }], &transport_conf
+        es = Elasticsearch::Client.new transport: transport
+      else
+        es = Elasticsearch::Client.new(:host => @config[:host], :port => @config[:port])
+      end
+    rescue
+      raise "Could not connect to Elasticsearch cluster: #{@config[:host]}:#{@config[:port]}"
+    end
+
+    return es
+  end
+
+  def get_indices
+    indexes = Array.new
+    start_str = @start_date.split('T').first.split('-').join('.')
+    s_year = start_str.split('.').first.to_i
+    s_mo = start_str.split('.')[1].to_i
+    s_day = start_str.split('.').last.to_i
+    start_date = Date.new(s_year, s_mo, s_day)
+
+    end_str = @end_date.split('T').first.split('-').join('.')
+    e_year = end_str.split('.').first.to_i
+    e_mo = end_str.split('.')[1].to_i
+    e_day = end_str.split('.').last.to_i
+    end_date = Date.new(e_year, e_mo, e_day)
+
+    (start_date..end_date).map do |day|
+      day = day.strftime('%Y.%m.%d')
+      @config[:index_prefixes].each do |prefix|
+        indexes << "#{prefix}#{day}"
+      end
+    end
+    return indexes
+  end
+
+  def run_query
+    queries = Array.new
+    queries << @query if @query
+    queries << @tags if @tags
+
+    if @start_date and @end_date
+      time_range = "@timestamp:[#{@start_date} TO #{@end_date}]"
+      queries << "#{time_range}"
+      indexes = get_indices
+    else
+      indexes [ '_all' ]
+    end
+
+    query = queries.join(' AND ')
+
+    ## Make sure each index exists
+    good_indexes = Array.new
+    unless indexes.include?('_all')
+      indexes.each do |index|
+        good_indexes << index if @es_conn.indices.exists index: index
+      end
+      indexes = good_indexes
+    else
+      indexes = [ '_all' ]
+    end
+
+    puts "Using these indices: #{indexes.join(',')}" if $debug
+
+    index_str = indexes.join(',')
+    res = @es_conn.search index: index_str, q: query, search_type: 'scan', scroll: @config[:scroll_time], size: @config[:scroll_size], df: 'message'
+    scroll_id = res['_scroll_id']
+
+    @scroll_ids << res['_scroll_id']
+    @num_results = res['hits']['total']
+    puts "Found #{@num_results} results" if @config[:print]
+
+    puts res.inspect if $debug
+
+    if @config[:output]
+      bar = ProgressBar.new(@num_results) if @config[:print]
+      hit_list = Array.new
+      total_lines = 0 if $debug
+      while true
+        res['hits']['hits'].each do |hit|
+          bar.increment! if @config[:print]
+          hit_list << hit
+          if hit_list.length % $flush_buffer == 0
+            flush_to_file hit_list.join("\n")
+            hit_list = Array.new
+          end
+        end
+        total_lines += res['hits']['hits'].length if $debug
+
+        # Continue scroll through data
+        begin
+          res = @es_conn.scroll scroll: @config[:scroll_time], body: scroll_id
+          scroll_id = res['_scroll_id']
+          @scroll_ids << res['_scroll_id']
+        rescue => e
+          puts res.inspect
+          raise e
+        end
+
+        begin
+          break if res['hits']['hits'].length < 1
+        rescue => e
+          raise e
+        end
+      end
+      flush_to_file hit_list
+    end
+
+    @query_finished = true
+    clean_scroll_ids
+  end
+
+  def clean_scroll_ids
+    ## Delete the scroll_ids to free up resources on the ES cluster
+    ## Have to use direct API call until elasticsearch-ruby supports this
+    @scroll_ids.uniq.each do |scroll|
+      puts "DELETE SCROLL:#{scroll}" if $debug
+      #puts
+      begin
+        Curl.delete("#{@config[:host]}:#{@config[:port]}/_search/scroll/#{scroll}")
+      rescue
+        puts "Delete failed" if $debug
+      end
+    end
+  end
+
+  end
+end

data/lib/stash-query/version.rb

@@ -0,0 +1,3 @@
+module Stashquery
+  VERSION = '0.1.2'
+end

data/stash-query.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'stash-query/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'stash-query'
+  s.version       = Stashquery::VERSION
+  s.date          = '2014-07-22'
+  s.homepage      = "https://github.com/robbydyer/stash-query"
+  s.summary       = "Gem for querying Logstash and exporting the results"
+  s.description   = "Gem for querying Logstash and exporting the results."
+  s.authors       = ["Robby Dyer"]
+  s.email         = 'robby.dyer@gmail.com'
+  s.licenses      = [ 'GPL-3.0' ]
+  s.files         = `git ls-files`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 
+  s.require_paths = [ 'lib' ]
+
+  s.add_runtime_dependency 'elasticsearch', '>=1.0.1'
+  s.add_runtime_dependency 'curb', '>= 0.8.5'
+  s.add_runtime_dependency 'faraday', '= 0.8.8'
+  s.add_runtime_dependency 'progress_bar'
+  s.add_runtime_dependency 'typhoeus', '= 0.6.6'
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: stash-query
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Robby Dyer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.5
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+- !ruby/object:Gem::Dependency
+  name: progress_bar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.6
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gem for querying Logstash and exporting the results.
+email: robby.dyer@gmail.com
+executables:
+- stash-query
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- bin/stash-query
+- lib/stash-query.rb
+- lib/stash-query/query.rb
+- lib/stash-query/version.rb
+- stash-query.gemspec
+homepage: https://github.com/robbydyer/stash-query
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Gem for querying Logstash and exporting the results
+test_files: []