lstash 0.2.0 → 1.0.0

data/lib/lstash/client.rb

@@ -1,49 +1,66 @@
-require 'logger'
-require 'date'
-require 'hashie'
+require "logger"
+require "date"
 
 class NullLogger < Logger
-  def initialize(*args); end
-  def add(*args, &block); end
+  def initialize(*args)
+  end
+
+  def add(*args, &block)
+  end
 end
 
 module Lstash
-
   class Client
-
     class ConnectionError < StandardError; end
 
-    PER_PAGE = 5000.freeze # best time, lowest resource usage
-    DEFAULT_COUNT_STEP = 3600.freeze # 1 hour
-    DEFAULT_GREP_STEP  = 120.freeze  # 2 minutes
+    class ShardMismatchError < StandardError; end
+
+    PER_PAGE = 5000 # best time, lowest resource usage
+    COUNT_STEP = 3600 # 1 hours
+    GREP_STEP = 3600 # 1 hour
 
     def initialize(es_client, options = {})
       raise ConnectionError, "No elasticsearch client specified" if es_client.nil?
 
       @es_client = es_client
-      @logger    = options[:logger] || (options[:debug] ? debug_logger : NullLogger.new)
+      @logger = options[:logger] || (options[:debug] ? debug_logger : NullLogger.new)
+      @wildcard = options[:wildcard]
     end
 
-    def count(query, step = DEFAULT_COUNT_STEP)
+    def count(query)
       @logger.debug "count from=#{query.from} to=#{query.to}"
 
       count = 0
-      query.each_period(step) do |index, hour_query|
-        count += count_messages(index, hour_query)
+      use_wildcard = @wildcard.nil? ? true : @wildcard
+      if use_wildcard
+        count = count_messages(query.wildcard_indices, query)
+      else
+        count = 0
+        query.each_period(COUNT_STEP) do |index, hour_query|
+          count += count_messages(index, hour_query)
+        end
       end
       @logger.debug "total count=#{count}"
       count
     end
 
-    def grep(query, step = DEFAULT_GREP_STEP)
+    def grep(query)
       @logger.debug "grep from=#{query.from} to=#{query.to}"
 
       count = 0
-      query.each_period(step) do |index, hour_query|
-        grep_messages(index, hour_query) do |message|
+      use_wildcard = @wildcard.nil? ? false : @wildcard
+      if use_wildcard
+        grep_messages(query.wildcard_indices, query) do |message|
           count += 1
           yield message if block_given?
         end
+      else
+        query.each_period(GREP_STEP) do |index, hour_query|
+          grep_messages(index, hour_query) do |message|
+            count += 1
+            yield message if block_given?
+          end
+        end
       end
 
       @logger.debug "total count=#{count}"
@@ -53,12 +70,10 @@ module Lstash
     private
 
     def count_messages(index, query)
-      result = Hashie::Mash.new @es_client.send(:count,
-        index: index,
-        body:  query.filter
-      )
-      @logger.debug "count index=#{index} from=#{query.from} to=#{query.to} count=#{result['count']}"
-      result['count']
+      result = @es_client.count(index: index, body: {query: query.filter})
+      validate_shards!(result["_shards"])
+      @logger.debug "count index=#{index} from=#{query.from.utc} to=#{query.to.utc} count=#{result["count"]}"
+      result["count"]
     end
 
     def grep_messages(index, query)
@@ -66,37 +81,43 @@ module Lstash
       scroll_params = {}
       offset = 0
       method = :search
-      while (messages.nil? || messages.count > 0) do
-        result = Hashie::Mash.new @es_client.send(method, {
-          index:  index,
-          scroll: '5m',
-          body:   query.search(offset, PER_PAGE),
-        }.merge(scroll_params))
+      while messages.nil? || messages.count > 0
+        result = @es_client.send(method, {
+          index: index,
+          scroll: "5m",
+          body: (method == :search) ? query.search(offset, PER_PAGE) : scroll_params
+        })
 
-        messages = result.hits.hits
+        validate_shards!(result["_shards"])
 
+        messages = result["hits"]["hits"]
         offset += messages.count
-        scroll_params = {scroll_id: result._scroll_id}
+        scroll_params = {scroll_id: result["_scroll_id"]}
 
         messages.each do |h|
-          next if h.fields.nil?
-          yield h.fields.message if block_given?
+          next if h["_source"].nil?
+          yield h["_source"]["message"] if block_given?
         end
 
         method = :scroll
+
+        @logger.debug "grep index=#{index} from=#{query.from.utc} to=#{query.to.utc} count=#{offset}"
       end
-      @logger.debug "grep index=#{index} from=#{query.from} to=#{query.to} count=#{offset}"
-      Hashie::Mash.new @es_client.clear_scroll(scroll_params)
+      @es_client.clear_scroll(body: scroll_params) unless scroll_params.empty?
     end
 
     def debug_logger
-      logger = Logger.new(STDERR)
+      logger = Logger.new($stderr)
       logger.formatter = proc do |severity, datetime, progname, msg|
         "#{datetime} #{msg}\n"
       end
       logger
     end
 
+    def validate_shards!(shards)
+      if shards["total"] != shards["successful"]
+        raise ShardMismatchError, "Shard mismatch: total: #{shards["total"]}, successful: #{shards["successful"]}"
+      end
+    end
   end
-
 end

data/lib/lstash/query.rb

@@ -1,47 +1,50 @@
-require 'time'
-require 'date'
-require 'ostruct'
+require "time"
+require "date"
+require "ostruct"
 
 module Lstash
-
   class Query
-
     class FormatError < StandardError; end
+
     class QueryMissing < StandardError; end
 
-    LOGSTASH_PREFIX = 'logstash-'.freeze
-    WILDCARD_QUERY  = '*'.freeze
+    LOGSTASH_PREFIX = "logstash-".freeze
+    WILDCARD_QUERY = "*".freeze
 
     attr_accessor :from, :to
 
     def initialize(query_string = nil, arguments = {})
       @query_string = query_string
 
-      @anchor = time_parse(arguments[:anchor], 'today')
-      @from   = time_parse(arguments[:from],   'today')
-      @to     = time_parse(arguments[:to],     'now')
+      @anchor = time_parse(arguments[:anchor], "today")
+      @from = time_parse(arguments[:from], "yesterday")
+      @to = time_parse(arguments[:to], "today")
 
       @to = Time.now if @to > Time.now # prevent accessing non-existing times / indices
     end
 
     def index_name(date)
-      "#{LOGSTASH_PREFIX}#{date.strftime('%Y.%m.%d')}"
+      "#{LOGSTASH_PREFIX}#{date.strftime("%Y.%m.%d")}"
+    end
+
+    def wildcard_indices
+      "logstash-*"
     end
 
     def search(from, size)
       {
-        sort:   sort_order,
-        fields: %w(message),
-        query:  filter,
-        from:   from,
-        size:   size
+        sort: sort_order,
+        _source: %w[message],
+        query: filter,
+        from: from,
+        size: size
       }
     end
 
     def filter
       {
-        filtered: {
-          query:  es_query,
+        bool: {
+          must: es_query,
           filter: es_filter
         }
       }
@@ -52,32 +55,37 @@ module Lstash
       time_iterate(@from.utc, @to.utc - 1, step) do |start_at|
         yield index_name(start_at.to_date),
               Query.new(@query_string,
-                        anchor: @anchor,
-                        from:   start_at,
-                        to:     start_at + step)
+                anchor: @anchor,
+                from: start_at,
+                to: start_at + step)
       end
     end
 
     private
 
     def time_iterate(start_time, end_time, step, &block)
-      begin
+      while start_time < end_time
         yield(start_time)
-      end while (start_time += step) < end_time
+        start_time += step
+      end
     end
 
     def time_parse(time_or_string, default)
       return time_or_string if time_or_string.is_a? Time
-      time_string = time_or_string.strip rescue nil
+      time_string = begin
+        time_or_string.strip
+      rescue
+        nil
+      end
       time_string ||= default
       case time_string
-      when 'firstday'
+      when "firstday"
         midnight_at_beginning_of_month
-      when 'now'
+      when "now"
         Time.now
-      when 'today'
+      when "today"
         midnight_today
-      when 'yesterday'
+      when "yesterday"
         midnight_yesterday
       else
         Time.parse(time_string)
@@ -87,66 +95,37 @@ module Lstash
     end
 
     def query_string
-      q = @query_string.dup.strip rescue ''
+      q = begin
+        @query_string.dup.strip
+      rescue
+        ""
+      end
       q = WILDCARD_QUERY if q.empty?
       q
     end
 
     def sort_order
       # return results in order of ascending timestamp
-      [ { '@timestamp' => { order: 'asc' } } ]
+      [{"@timestamp" => {order: "asc"}}]
     end
 
     def es_query
-      {
-        bool: {
-          should: [
-            {
-              query_string: {
-                query: query_string
-              }
-            }
-          ]
+      [
+        {
+          query_string: {
+            query: query_string
+          }
         }
-      }
+      ]
     end
 
     def es_filter
       {
-        bool: {
-          must: [
-            range: {
-              '@timestamp' => {
-                gte: to_msec(from),
-                lt:  to_msec(to)
-              }
-            },
-            # fquery: {
-            #   query: {
-            #     query_string: {
-            #       query: query_string
-            #     }
-            #   }
-            # }
-          ],
-          # must_not: [
-          #   fquery: {
-          #     query: {
-          #       query_string: {
-          #         query: query_string
-          #       }
-          #     }
-          #   }
-          # ],
-          # should: [
-          #   fquery: {
-          #     query: {
-          #       query_string: {
-          #         query: query_string
-          #       }
-          #     }
-          #   }
-          # ]
+        range: {
+          "@timestamp" => {
+            gte: to_msec(from),
+            lt: to_msec(to)
+          }
         }
       }
     end
@@ -161,7 +140,7 @@ module Lstash
     end
 
     def midnight_yesterday
-      (Date.today-1).to_time
+      (Date.today - 1).to_time
     end
 
     def anchor_time
@@ -171,7 +150,5 @@ module Lstash
     def to_msec(time)
       time.to_i * 1000
     end
-
   end
-
 end

data/lib/lstash/version.rb

@@ -1,3 +1,3 @@
 module Lstash
-  VERSION = "0.2.0"
+  VERSION = "1.0.0"
 end

data/lib/lstash.rb

@@ -1,7 +1,7 @@
-require 'elasticsearch'
-require 'lstash/version'
-require 'lstash/query'
-require 'lstash/client'
+require "elasticsearch"
+require "lstash/version"
+require "lstash/query"
+require "lstash/client"
 
 module Lstash
   # Your code goes here...

data/lstash.gemspec

@@ -1,23 +1,21 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'lstash/version'
+require "lstash/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "lstash"
-  spec.version       = Lstash::VERSION
-  spec.authors       = ["Klaas Jan Wierenga"]
-  spec.email         = ["k.j.wierenga@gmail.com"]
-  spec.description   = %q{Count or grep log messages in a specified time range from a Logstash Elasticsearch server.}
-  spec.summary       = %q{The lstash gem allows you to count or grep log messages in a specific time range from a Logstash Elasticsearch server. }
-  spec.homepage      = "https://github.com/kjwierenga/lstash"
-  spec.license       = "MIT"
+  spec.name = "lstash"
+  spec.version = Lstash::VERSION
+  spec.authors = ["Klaas Jan Wierenga"]
+  spec.email = ["k.j.wierenga@kerkdienstgemist.nl"]
+  spec.description = "Count or grep log messages in a specified time range from a Logstash Elasticsearch server."
+  spec.summary = "The lstash gem allows you to count or grep log messages in a specific time range from a Logstash Elasticsearch server. "
+  spec.homepage = "https://github.com/kdgm/lstash"
+  spec.license = "MIT"
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.files = `git ls-files`.split($/)
+  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.extra_rdoc_files = [ 'LICENSE.txt', 'README.md', 'CHANGELOG.md' ]
+  spec.extra_rdoc_files = ["LICENSE.txt", "README.md", "CHANGELOG.md"]
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake", "~> 10.3.2"
@@ -25,9 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec-its", "~> 1.0.1"
   spec.add_development_dependency "timecop", "~> 0.7.1"
 
-  spec.add_dependency "typhoeus", "~> 1.4.0"
-  spec.add_dependency "elasticsearch", "~> 0.4"
-  spec.add_dependency "hashie", "~> 4.1.0"
+  spec.add_dependency "elasticsearch", "~> 7.17.7"
   spec.add_dependency "thor", "~> 0.20.3"
-  spec.add_dependency "faraday", "~> 0.17.4"
 end

data/spec/lstash/cli_spec.rb

@@ -1,16 +1,15 @@
-require 'spec_helper'
-require 'lstash/cli'
+require "spec_helper"
+require "lstash/cli"
 
-class Lstash::CLI < Thor
+class Lstash::CLI < Lstash::CLIBase
   def self.exit_on_failure?
     false
   end
 end
 
 describe Lstash::CLI do
-
   before(:all) do
-    Timecop.freeze('2014-08-01 14:58')
+    Timecop.freeze("2014-08-01 14:58")
   end
   after(:all) do
     Timecop.return
@@ -19,17 +18,16 @@ describe Lstash::CLI do
   context "options" do
     subject { Lstash::CLI.options(args) }
 
-    let(:args) { %w(extract --time from --to to --one --two --three --four) }
+    let(:args) { %w[extract --time from --to to --one --two --three --four] }
 
     its(:keys) { should eq args }
   end
 
   context "count" do
-
     context "with full URI" do
-      let(:args) { %w(count "*" --es-url http://localhost:9200) }
+      let(:args) { %w[count "*" --es-url http://localhost:9200] }
       it "should succeed" do
-        client = double('client')
+        client = double("client")
 
         allow(Lstash::Client).to receive(:new).and_return(client)
         allow(client).to receive(:count).and_return(1000)
@@ -42,10 +40,10 @@ describe Lstash::CLI do
     end
 
     context "with valid arguments" do
-      let(:args) { %w(count "program:haproxy" --es-url localhost) }
+      let(:args) { %w[count "program:haproxy" --es-url localhost] }
 
       it "should succeed" do
-        client = double('client')
+        client = double("client")
 
         allow(Lstash::Client).to receive(:new).and_return(client)
         allow(client).to receive(:count).and_return(100)
@@ -57,7 +55,7 @@ describe Lstash::CLI do
     end
 
     context "with invalid --es-url" do
-      let(:args) { %w(count "program:haproxy" --es-url '') }
+      let(:args) { %w[count "program:haproxy" --es-url ''] }
 
       it "should print error message" do
         output = capture_stderr { Lstash::CLI.start(args) }
@@ -67,7 +65,7 @@ describe Lstash::CLI do
     end
 
     context "without query" do
-      let(:args) { %w() }
+      let(:args) { %w[] }
       it "should print help message" do
         output = capture_stdout { Lstash::CLI.start(args) }
 
@@ -76,35 +74,35 @@ describe Lstash::CLI do
     end
 
     context "with anchor date" do
-      let(:args) { %w(count program:haproxy --from firstday --to today --anchor yesterday) }
+      let(:args) { %w[count program:haproxy --from firstday --to today --anchor yesterday --no-wildcard] }
 
       it "should return correct count" do
-        es_client = double('es_client')
+        es_client = double("es_client")
 
         allow(Elasticsearch::Client).to receive(:new) { es_client }
+        allow_any_instance_of(Lstash::Client).to receive(:validate_shards!).and_return(true)
 
-        expect(es_client).to receive(:count).exactly(31 * 24).times.and_return(count:100)
+        expect(es_client).to receive(:count).exactly(31 * 24).times.and_return("count" => 100)
 
         output = capture_stdout { Lstash::CLI.start(args) }
-        expect(output).to match("#{31 * 24 * 100}")
+        expect(output).to match((31 * 24 * 100).to_s)
       end
     end
 
     context "without anchor date" do
-      let(:args) { %w(count program:haproxy --from yesterday --to today) }
+      let(:args) { %w[count program:haproxy --from yesterday --to today --no-wildcard] }
 
       it "should return correct count" do
-        es_client = double('es_client')
+        es_client = double("es_client")
 
         allow(Elasticsearch::Client).to receive(:new) { es_client }
+        allow_any_instance_of(Lstash::Client).to receive(:validate_shards!).and_return(true)
 
-        expect(es_client).to receive(:count).exactly(24).times.and_return(count:100)
+        expect(es_client).to receive(:count).exactly(24).times.and_return("count" => 100)
 
         output = capture_stdout { Lstash::CLI.start(args) }
-        expect(output).to match("#{24 * 100}")
+        expect(output).to match((24 * 100).to_s)
       end
     end
-
   end
-
 end

data/spec/lstash/client_spec.rb

@@ -1,30 +1,30 @@
-require 'spec_helper'
-require 'lstash/client'
+require "spec_helper"
+require "lstash/client"
 
 describe Lstash::Client do
-
-  let(:es_client) { double('es_client') }
-  subject { Lstash::Client.new(es_client) }
+  let(:es_client) { double("es_client") }
+  subject { Lstash::Client.new(es_client, wildcard: false) }
+  before do
+    allow(subject).to receive(:validate_shards!).and_return(true)
+  end
 
   it "should initialize properly" do
     expect(subject).not_to be nil
   end
 
   context "with query" do
-
     context "count" do
       it "should return number of messages matching query" do
-
-        query = Lstash::Query.new("*", from: Time.parse("2014-10-10 00:00"), to: Time.parse("2014-10-10 07:00"))
+        query = Lstash::Query.new("*", from: "2014-10-10 00:00", to: "2014-10-10 07:00")
 
         allow(es_client).to receive(:count).and_return(
-          { 'count' => 100 },
-          { 'count' => 200 },
-          { 'count' => 300 },
-          { 'count' => 400 },
-          { 'count' => 500 },
-          { 'count' => 600 },
-          { 'count' => 700 }
+          {"count" => 100},
+          {"count" => 200},
+          {"count" => 300},
+          {"count" => 400},
+          {"count" => 500},
+          {"count" => 600},
+          {"count" => 700}
         )
 
         expect(subject.count(query)).to eq 2800
@@ -33,35 +33,31 @@ describe Lstash::Client do
 
     context "grep" do
       it "should return the messages matching the query" do
-
-        query = Lstash::Query.new("*", from: Time.parse("2014-10-10 00:00"), to: Time.parse("2014-10-10 07:00"))
-      
-        expect(es_client).to receive(:search).and_return(
-          hits(%w(1)),
-          hits(%w(2 2)),
-          hits(%w(3 3 3)),
-          hits(%w(4 4 4 4)),
-          hits(%w(5 5 5 5 5)),
-          hits(%w(6 6 6 6 6 6)),
-          hits(%w(7 7 7 7 7 7 7))
+        query = Lstash::Query.new("*", from: "2014-10-10 00:00", to: "2014-10-10 07:00")
+
+        allow(es_client).to receive(:search).and_return(
+          hits(%w[1]),
+          hits(%w[2 2]),
+          hits(%w[3 3 3]),
+          hits(%w[4 4 4 4]),
+          hits(%w[5 5 5 5 5]),
+          hits(%w[6 6 6 6 6 6]),
+          hits(%w[7 7 7 7 7 7 7])
         )
 
         allow(es_client).to receive(:scroll).and_return(hits([]))
 
-        allow(es_client).to receive(:clear_scroll)
-
-        expect(subject.grep(query, 3600)).to eq 28
+        expect(es_client).to receive(:clear_scroll).exactly(7).times
+        expect(subject.grep(query)).to eq 28
       end
     end
-
   end
 
   def hits(messages)
     {
-      hits: {
-        hits: messages.map { |m| { fields: { message: m }}}
+      "hits" => {
+        "hits" => messages.map { |m| {"_source" => {"message" => m}} }
       }
     }
   end
-
 end