lstash 0.0.9 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NTgxYmI5NWQ2ZmE3OGY1NDE0ZGNjN2M1ZGQ2ZWYxNGI5ODY0MGY1NQ==
-  data.tar.gz: !binary |-
-    ZTgzZmQ0YTU3MTAwZDA1MWI1MjNmMjM0M2M5NWY1Mjk5NWRmMTM4OA==
+SHA256:
+  metadata.gz: 0f41d0f4233ebb92ad410bbc04a55f8b39d7205e576283677ca2de977c24aea5
+  data.tar.gz: f038956a733448fa7902e9e9c8aaed5a893881ec3154fa39a936c467a58e8e00
 SHA512:
-  metadata.gz: !binary |-
-    YThlNDM0OTNjZTNhYWI3ODc4MDc5ZjUyOGJhMTBmZTc4YjI3Nzc2YTZiMjFl
-    ZDk0YTQ3NGNkZDNmMThhN2Y2ODVmOWY5NTliYzQ3MTZmNzc2YzEzZTNmZjU4
-    YjAzMTMxNzEwZDgzNDdjMTM2ZTRlYjZjNjZmM2VlMzJkMTE0NDY=
-  data.tar.gz: !binary |-
-    N2Y5NWI5ZDdiNGNjOTFhMTJkMjJjMzVmZmExZTQ1NjFhZjMzNGU3MDdjMDFi
-    ZDE4Y2U5MGUyYzY3NTI5N2MyMDE5MDY2NjVhYWVhZTY2Mzk5YmFmNmVkM2Rm
-    ZjI2ZmYwOTE5NWMxNmI5YTlmYTdjMjRkY2QxZDRlOWNmMjAzNDU=
+  metadata.gz: d0a803e99283cf78ccceb1f24ab266cb2ac6b650574c9272f2a13a760a9ea021a3c7be54de3e6c68c5ffa9b9ec22a1afc5aa262502c2a45f98b8aaf1eced27a2
+  data.tar.gz: 68209058f27b409c9d2804431cbf7e7426445e084854a26c49640aaf7b5a466f7019804458338bf73fb14ccebf1c7302bc4014fa7abe48c9501900fa2ab475d7

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.log

data/.ruby-version

@@ -1 +1 @@
-ruby-1.9.3
+2.4.6

data/CHANGELOG.md

@@ -1,3 +1,34 @@
+## Release 0.2.0
+Merge branch 'feature/fix/hashie-warnings' into develop
+- [fix] pin faraday to a compatible version
+- [enh] upgrade to ruby 2.4(.6)
+- [fix] suppress Hashie warnings; it would generate a warning for each log line (on stdout) leading to very large output
+
+### 0.1.4 / 2015-05-29
+
+Enhancements
+
+* Change default time steps from 1 hour to 120 seconds to aid in grepping from datasets with many events per minute.
+
+### 0.1.3 / 2015-05-28
+
+Enhancements
+
+* Remove troublesome development dependencies (autotest).
+
+### 0.1.2 / 2015-01-02
+
+Enhancements
+
+* Double Elasticsearch::Client request timeout to 120 seconds to prevent Timeout::Error (Faraday::TimeoutError).
+
+### 0.1.1 / 2014-10-13
+
+Enhancements
+
+* Refactor to iterate over hour periods instead of full indices. This improves the performance a lot.
+* Increase timeout to prevent Faraday::TimeoutError exceptions.
+
 ### 0.0.9 / 2014-09-22
 
 Enhancements

data/README.md

@@ -24,7 +24,7 @@ Or install it yourself as:
 	  lstash count QUERY
 
 	Description:
-	  Count log messages matching the QUERY from Logstash and output this count to stdout. QUERY can use Apache Lucene query 
+	  Count log messages matching the QUERY from Logstash and output this count to stdout. QUERY can use Apache Lucene query
 	  parser syntax.
 
 	  Example to count the number of HAProxy log messages in yesterdays month.
@@ -82,38 +82,38 @@ Assuming today is Sep 1 2014. Count all haproxy log messages in the previous mon
 
 	lstash count program:haproxy --anchor yesterday --from firstday --to today -d
 	time range: [2014-08-01 00:00:00 +0200..2014-09-01 00:00:00 +0200]
-	logstash-2014.07.31: 1 
-	logstash-2014.08.01: 13 
-	logstash-2014.08.02: 14 
-	logstash-2014.08.03: 1654 
-	logstash-2014.08.04: 6 
-	logstash-2014.08.05: 20 
-	logstash-2014.08.06: 219 
-	logstash-2014.08.07: 32 
-	logstash-2014.08.08: 14 
-	logstash-2014.08.09: 28 
-	logstash-2014.08.10: 799 
-	logstash-2014.08.11: 18 
-	logstash-2014.08.12: 8 
-	logstash-2014.08.13: 23 
-	logstash-2014.08.14: 25 
-	logstash-2014.08.15: 69 
-	logstash-2014.08.16: 19 
-	logstash-2014.08.17: 1160 
-	logstash-2014.08.18: 284 
-	logstash-2014.08.19: 61 
-	logstash-2014.08.20: 26 
-	logstash-2014.08.21: 16 
-	logstash-2014.08.22: 145 
-	logstash-2014.08.23: 72 
-	logstash-2014.08.24: 792 
-	logstash-2014.08.25: 31 
-	logstash-2014.08.26: 33 
-	logstash-2014.08.27: 51 
-	logstash-2014.08.28: 8 
-	logstash-2014.08.29: 23 
-	logstash-2014.08.30: 25 
-	logstash-2014.08.31: 69 
+	logstash-2014.07.31: 1
+	logstash-2014.08.01: 13
+	logstash-2014.08.02: 14
+	logstash-2014.08.03: 1654
+	logstash-2014.08.04: 6
+	logstash-2014.08.05: 20
+	logstash-2014.08.06: 219
+	logstash-2014.08.07: 32
+	logstash-2014.08.08: 14
+	logstash-2014.08.09: 28
+	logstash-2014.08.10: 799
+	logstash-2014.08.11: 18
+	logstash-2014.08.12: 8
+	logstash-2014.08.13: 23
+	logstash-2014.08.14: 25
+	logstash-2014.08.15: 69
+	logstash-2014.08.16: 19
+	logstash-2014.08.17: 1160
+	logstash-2014.08.18: 284
+	logstash-2014.08.19: 61
+	logstash-2014.08.20: 26
+	logstash-2014.08.21: 16
+	logstash-2014.08.22: 145
+	logstash-2014.08.23: 72
+	logstash-2014.08.24: 792
+	logstash-2014.08.25: 31
+	logstash-2014.08.26: 33
+	logstash-2014.08.27: 51
+	logstash-2014.08.28: 8
+	logstash-2014.08.29: 23
+	logstash-2014.08.30: 25
+	logstash-2014.08.31: 69
 	5633
 
 ## Using lstash as a gem in your project
@@ -145,6 +145,28 @@ Usage:
 	  puts message
 	end
 
+## Publishing the gem to RubyGems.org
+
+1. Build the gem
+
+```
+$ gem build lstash.gem
+  Successfully built RubyGem
+  Name: lstash
+  Version: 0.2.0
+  File: lstash-0.2.0.gem
+```
+
+2. Pushing your gem to RubyGems.org
+
+```
+gem push lstash-0.2.0.gem
+  Pushing gem to RubyGems.org...
+  Successfully registered gem: lstash (0.2.0)
+```
+
+See [RubyGems.org documention](https://guides.rubygems.org/) for more info.
+
 ## Contributing
 
 1. Fork it

data/bin/lstash

@@ -2,4 +2,6 @@
 
 require 'lstash/cli'
 
+# Suppress Hashie warnings
+Hashie.logger = Logger.new(nil)
 Lstash::CLI.start(ARGV)

data/lib/lstash/cli.rb

@@ -9,6 +9,8 @@ require 'lstash/client'
 
 module Lstash
 
+  TRANSPORT_REQUEST_TIMEOUT = 120.freeze # 2 minute request timeout
+
   class CLI < Thor
 
     class_option :from,   :banner => 'start of time range', :aliases => '-f', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
@@ -45,7 +47,7 @@ module Lstash
     desc "count QUERY", "count number of log messages matching the QUERY"
     def count(query_string)
       run_command(query_string) do |es_client, query|
-        count  = Lstash::Client.new(es_client, options).count(query)
+        count = Lstash::Client.new(es_client, options).count(query)
         puts count
       end
     end
@@ -55,14 +57,15 @@ module Lstash
     def run_command(query_string)
       es_client = ::Elasticsearch::Client.new(
         url: options[:es_url] || ENV['ES_URL'] || 'localhost',
-        log: !!ENV['DEBUG']
+        log: !!ENV['DEBUG'],
+        transport_options: { request: { timeout: TRANSPORT_REQUEST_TIMEOUT } }
       )
       query  = Lstash::Query.new(query_string, options)
 
       yield es_client, query
 
     rescue Exception => e
-      raise Thor::Error.new(e.message)
+      options[:debug] ? raise(e) : raise(Thor::Error.new(e.message))
     end
 
     protected

data/lib/lstash/client.rb

@@ -13,7 +13,9 @@ module Lstash
 
     class ConnectionError < StandardError; end
 
-    PER_PAGE = 5000 # best time, lowest resource usage
+    PER_PAGE = 5000.freeze # best time, lowest resource usage
+    DEFAULT_COUNT_STEP = 3600.freeze # 1 hour
+    DEFAULT_GREP_STEP  = 120.freeze  # 2 minutes
 
     def initialize(es_client, options = {})
       raise ConnectionError, "No elasticsearch client specified" if es_client.nil?
@@ -22,25 +24,30 @@ module Lstash
       @logger    = options[:logger] || (options[:debug] ? debug_logger : NullLogger.new)
     end
 
-    def count(query)
-      @logger.debug "time range: [%s..%s]" % [query.time_range.from, query.time_range.to]
+    def count(query, step = DEFAULT_COUNT_STEP)
+      @logger.debug "count from=#{query.from} to=#{query.to}"
 
       count = 0
-      query.indices.each do |index|
-        count += count_messages(index, query)
+      query.each_period(step) do |index, hour_query|
+        count += count_messages(index, hour_query)
       end
-
+      @logger.debug "total count=#{count}"
       count
     end
 
-    def grep(query)
-      @logger.debug "time range: [%s..%s]" % [query.time_range.from, query.time_range.to]
+    def grep(query, step = DEFAULT_GREP_STEP)
+      @logger.debug "grep from=#{query.from} to=#{query.to}"
 
-      query.indices.each do |index|
-        grep_messages(index, query) do |message|
+      count = 0
+      query.each_period(step) do |index, hour_query|
+        grep_messages(index, hour_query) do |message|
+          count += 1
           yield message if block_given?
         end
       end
+
+      @logger.debug "total count=#{count}"
+      count
     end
 
     private
@@ -48,9 +55,9 @@ module Lstash
     def count_messages(index, query)
       result = Hashie::Mash.new @es_client.send(:count,
         index: index,
-        body:  query.body[:query]
+        body:  query.filter
       )
-      @logger.debug "count #{index}: #{result['count']} "
+      @logger.debug "count index=#{index} from=#{query.from} to=#{query.to} count=#{result['count']}"
       result['count']
     end
 
@@ -61,9 +68,9 @@ module Lstash
       method = :search
       while (messages.nil? || messages.count > 0) do
         result = Hashie::Mash.new @es_client.send(method, {
-          index: index,
-          scroll: '10m',
-          body: query.body.merge(from: offset, size: PER_PAGE),
+          index:  index,
+          scroll: '5m',
+          body:   query.search(offset, PER_PAGE),
         }.merge(scroll_params))
 
         messages = result.hits.hits
@@ -72,19 +79,20 @@ module Lstash
         scroll_params = {scroll_id: result._scroll_id}
 
         messages.each do |h|
+          next if h.fields.nil?
           yield h.fields.message if block_given?
         end
 
         method = :scroll
       end
-      @logger.debug "grep #{index}: #{offset}"
+      @logger.debug "grep index=#{index} from=#{query.from} to=#{query.to} count=#{offset}"
       Hashie::Mash.new @es_client.clear_scroll(scroll_params)
     end
 
     def debug_logger
       logger = Logger.new(STDERR)
       logger.formatter = proc do |severity, datetime, progname, msg|
-        "#{msg}\n"
+        "#{datetime} #{msg}\n"
       end
       logger
     end

data/lib/lstash/query.rb

@@ -9,11 +9,13 @@ module Lstash
     class FormatError < StandardError; end
     class QueryMissing < StandardError; end
 
-    LOGSTASH_PREFIX = 'logstash-'
-    WILDCARD_QUERY = '*'
+    LOGSTASH_PREFIX = 'logstash-'.freeze
+    WILDCARD_QUERY  = '*'.freeze
 
-    def initialize(query = nil, arguments = {})
-      @query = query
+    attr_accessor :from, :to
+
+    def initialize(query_string = nil, arguments = {})
+      @query_string = query_string
 
       @anchor = time_parse(arguments[:anchor], 'today')
       @from   = time_parse(arguments[:from],   'today')
@@ -22,38 +24,51 @@ module Lstash
       @to = Time.now if @to > Time.now # prevent accessing non-existing times / indices
     end
 
-    def time_range
-      OpenStruct.new(from: @from, to: @to)
-    end
-
-    def date_range
-      (@from.utc.to_date .. @to.utc.to_date)
+    def index_name(date)
+      "#{LOGSTASH_PREFIX}#{date.strftime('%Y.%m.%d')}"
     end
 
-    def indices
-      date_range.map { |d| "#{LOGSTASH_PREFIX}#{d.strftime('%Y.%m.%d')}" }
-    end
-
-    def body
+    def search(from, size)
       {
-        sort: sort_order,
-
+        sort:   sort_order,
         fields: %w(message),
+        query:  filter,
+        from:   from,
+        size:   size
+      }
+    end
 
-        # return in order of ascending timestamp
-        query: {
-          filtered: {
-            query:  es_query,
-            filter: es_filter
-          }
+    def filter
+      {
+        filtered: {
+          query:  es_query,
+          filter: es_filter
         }
       }
     end
 
+    def each_period(step, &block)
+      # iterate over the whole range in blocks of the specified period
+      time_iterate(@from.utc, @to.utc - 1, step) do |start_at|
+        yield index_name(start_at.to_date),
+              Query.new(@query_string,
+                        anchor: @anchor,
+                        from:   start_at,
+                        to:     start_at + step)
+      end
+    end
+
     private
 
-    def time_parse(time_string, default)
-      time_string = time_string.strip rescue nil
+    def time_iterate(start_time, end_time, step, &block)
+      begin
+        yield(start_time)
+      end while (start_time += step) < end_time
+    end
+
+    def time_parse(time_or_string, default)
+      return time_or_string if time_or_string.is_a? Time
+      time_string = time_or_string.strip rescue nil
       time_string ||= default
       case time_string
       when 'firstday'
@@ -71,13 +86,14 @@ module Lstash
       raise FormatError, "Invalid time format: #{time_string}"
     end
 
-    def query
-      q = @query.dup.strip rescue ''
+    def query_string
+      q = @query_string.dup.strip rescue ''
       q = WILDCARD_QUERY if q.empty?
       q
     end
 
     def sort_order
+      # return results in order of ascending timestamp
       [ { '@timestamp' => { order: 'asc' } } ]
     end
 
@@ -87,7 +103,7 @@ module Lstash
           should: [
             {
               query_string: {
-                query: query
+                query: query_string
               }
             }
           ]
@@ -101,14 +117,14 @@ module Lstash
           must: [
             range: {
               '@timestamp' => {
-                from: to_msec(time_range.from),
-                to:   to_msec(time_range.to)
+                gte: to_msec(from),
+                lt:  to_msec(to)
               }
             },
             # fquery: {
             #   query: {
             #     query_string: {
-            #       query: query
+            #       query: query_string
             #     }
             #   }
             # }
@@ -117,7 +133,7 @@ module Lstash
           #   fquery: {
           #     query: {
           #       query_string: {
-          #         query: query
+          #         query: query_string
           #       }
           #     }
           #   }
@@ -126,7 +142,7 @@ module Lstash
           #   fquery: {
           #     query: {
           #       query_string: {
-          #         query: query
+          #         query: query_string
           #       }
           #     }
           #   }

data/lib/lstash/version.rb

@@ -1,3 +1,3 @@
 module Lstash
-  VERSION = "0.0.9"
+  VERSION = "0.2.0"
 end

data/lstash.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["k.j.wierenga@gmail.com"]
   spec.description   = %q{Count or grep log messages in a specified time range from a Logstash Elasticsearch server.}
   spec.summary       = %q{The lstash gem allows you to count or grep log messages in a specific time range from a Logstash Elasticsearch server. }
-  spec.homepage      = "http://bitbucket.org/kjwierenga/lstash"
+  spec.homepage      = "https://github.com/kjwierenga/lstash"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($/)
@@ -20,16 +20,14 @@ Gem::Specification.new do |spec|
   spec.extra_rdoc_files = [ 'LICENSE.txt', 'README.md', 'CHANGELOG.md' ]
 
   spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rspec-its"
-  spec.add_development_dependency "rspec-autotest"
-  spec.add_development_dependency "autotest-standalone"
-  spec.add_development_dependency "autotest-fsevent"
-  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "rake", "~> 10.3.2"
+  spec.add_development_dependency "rspec", "~> 3.0.0"
+  spec.add_development_dependency "rspec-its", "~> 1.0.1"
+  spec.add_development_dependency "timecop", "~> 0.7.1"
 
-  spec.add_dependency "typhoeus"
+  spec.add_dependency "typhoeus", "~> 1.4.0"
   spec.add_dependency "elasticsearch", "~> 0.4"
-  spec.add_dependency "hashie"
-  spec.add_dependency "thor"
+  spec.add_dependency "hashie", "~> 4.1.0"
+  spec.add_dependency "thor", "~> 0.20.3"
+  spec.add_dependency "faraday", "~> 0.17.4"
 end

data/spec/lstash/cli_spec.rb

@@ -9,6 +9,13 @@ end
 
 describe Lstash::CLI do
 
+  before(:all) do
+    Timecop.freeze('2014-08-01 14:58')
+  end
+  after(:all) do
+    Timecop.return
+  end
+
   context "options" do
     subject { Lstash::CLI.options(args) }
 
@@ -25,9 +32,12 @@ describe Lstash::CLI do
         client = double('client')
 
         allow(Lstash::Client).to receive(:new).and_return(client)
-        allow(client).to receive(:count).and_return(100)
+        allow(client).to receive(:count).and_return(1000)
 
-        expect { Lstash::CLI.start(args) }.not_to raise_error
+        expect {
+          output = capture_stdout { Lstash::CLI.start(args) }
+          expect(output).to eq "1000\n"
+        }.not_to raise_error
       end
     end
 
@@ -52,7 +62,7 @@ describe Lstash::CLI do
       it "should print error message" do
         output = capture_stderr { Lstash::CLI.start(args) }
 
-        expect(output).to eq "the scheme http does not accept registry part: '':9200 (or bad hostname?)\n"
+        expect(output).to include("Failed to open TCP connection to '':9200")
       end
     end
 
@@ -68,52 +78,33 @@ describe Lstash::CLI do
     context "with anchor date" do
       let(:args) { %w(count program:haproxy --from firstday --to today --anchor yesterday) }
 
-      it "should succeed" do
-        Timecop.freeze('2014-08-01 14:58') do
-          es_client = double('es_client')
+      it "should return correct count" do
+        es_client = double('es_client')
 
-          allow(Elasticsearch::Client).to receive(:new) { es_client }
+        allow(Elasticsearch::Client).to receive(:new) { es_client }
 
-          expect(es_client).to receive(:count).with(satisfy { |args|
-            expect_time_range(args, [
-              Time.parse('2014-07-01').to_i*1000,
-              Time.parse('2014-08-01').to_i*1000
-            ])
-          }).exactly(32).times.and_return({count:1})
+        expect(es_client).to receive(:count).exactly(31 * 24).times.and_return(count:100)
 
-          Lstash::CLI.start(args)
-        end
+        output = capture_stdout { Lstash::CLI.start(args) }
+        expect(output).to match("#{31 * 24 * 100}")
       end
     end
 
     context "without anchor date" do
       let(:args) { %w(count program:haproxy --from yesterday --to today) }
 
-      it "should succeed" do
-        Timecop.freeze('2014-08-01 14:58') do
-          es_client = double('es_client')
+      it "should return correct count" do
+        es_client = double('es_client')
 
-          allow(Elasticsearch::Client).to receive(:new) { es_client }
+        allow(Elasticsearch::Client).to receive(:new) { es_client }
 
-          expect(es_client).to receive(:count).with(satisfy { |args|
-            expect_time_range(args, [
-              Time.parse('2014-07-31').to_i*1000,
-              Time.parse('2014-08-01').to_i*1000
-            ])
-          }).exactly(2).times.and_return({count:1})
+        expect(es_client).to receive(:count).exactly(24).times.and_return(count:100)
 
-          Lstash::CLI.start(args)
-        end
+        output = capture_stdout { Lstash::CLI.start(args) }
+        expect(output).to match("#{24 * 100}")
       end
     end
 
   end
 
-  private
-
-  def expect_time_range(args, time_range)
-    expect(args[:body][:filtered][:filter][:bool][:must].first[:range]['@timestamp'][:from]).to eq time_range.first
-    expect(args[:body][:filtered][:filter][:bool][:must].first[:range]['@timestamp'][:to]).to   eq time_range.last
-  end
-
 end

data/spec/lstash/client_spec.rb

@@ -12,38 +12,45 @@ describe Lstash::Client do
 
   context "with query" do
 
-    let(:query) { double('query', time_range: OpenStruct.new) }
-
     context "count" do
       it "should return number of messages matching query" do
-        allow(query).to receive(:indices).and_return (['logstash-2014-08-01', 'logstash-2014-08-02'])
-        allow(query).to receive(:body).and_return ({})
 
-        allow(es_client).to receive(:count).and_return({'count' => 100},{'count' => 100})
+        query = Lstash::Query.new("*", from: Time.parse("2014-10-10 00:00"), to: Time.parse("2014-10-10 07:00"))
+
+        allow(es_client).to receive(:count).and_return(
+          { 'count' => 100 },
+          { 'count' => 200 },
+          { 'count' => 300 },
+          { 'count' => 400 },
+          { 'count' => 500 },
+          { 'count' => 600 },
+          { 'count' => 700 }
+        )
 
-        expect(subject.count(query)).to eq 200
+        expect(subject.count(query)).to eq 2800
       end
     end
 
     context "grep" do
-      let(:query) { double('query', time_range: OpenStruct.new) }
-      
       it "should return the messages matching the query" do
-        allow(query).to receive(:indices).and_return (['logstash-2014-08-01', 'logstash-2014-08-02'])
-        allow(query).to receive(:body).and_return ({})
-        
-        allow(es_client).to receive(:search).and_return(
-          hits([
-            'this is the first log line',
-            'this is the second log line'
-          ])
+
+        query = Lstash::Query.new("*", from: Time.parse("2014-10-10 00:00"), to: Time.parse("2014-10-10 07:00"))
+      
+        expect(es_client).to receive(:search).and_return(
+          hits(%w(1)),
+          hits(%w(2 2)),
+          hits(%w(3 3 3)),
+          hits(%w(4 4 4 4)),
+          hits(%w(5 5 5 5 5)),
+          hits(%w(6 6 6 6 6 6)),
+          hits(%w(7 7 7 7 7 7 7))
         )
 
         allow(es_client).to receive(:scroll).and_return(hits([]))
 
         allow(es_client).to receive(:clear_scroll)
 
-        subject.grep(query)
+        expect(subject.grep(query, 3600)).to eq 28
       end
     end
 

data/spec/lstash/query_spec.rb

@@ -5,10 +5,10 @@ describe Lstash::Query do
 
   context "running on 2014-08-03" do
     let(:time)  { '2014-08-03 15:54:33' }
-    let(:query) { nil }
+    let(:query_string) { nil }
     let(:options) { {} }
 
-    subject { Lstash::Query.new(query, options) }
+    subject { Lstash::Query.new(query_string, options) }
 
     before { Timecop.freeze(Time.parse(time)) }
     after  { Timecop.return }
@@ -18,110 +18,171 @@ describe Lstash::Query do
     #   expect(subject).not_to be nil
     # end
 
-    its('time_range.from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
-    its('time_range.to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+    its('from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+    its('to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
 
-    its(:date_range) { should eq (Date.parse('2014-08-02')..Date.parse('2014-08-03')) }
+    # its(:date_range) { should eq (Date.parse('2014-08-02')..Date.parse('2014-08-03')) }
 
-    its(:indices) { should eq [ "logstash-2014.08.02", "logstash-2014.08.03" ] }
+    # its(:indices) { should eq [ "logstash-2014.08.02", "logstash-2014.08.03" ] }
 
     context "with specific query" do
-      let(:query) { 'program:haproxy' }
-      its(:query) { should eq query }
+      let(:query_string) { 'program:haproxy' }
+      its(:query_string) { should eq query_string }
     end
 
     context "without query" do
-      let(:query) { nil }
-      its(:query) { should eql '*' }
+      let(:query_string) { nil }
+      its(:query_string) { should eql '*' }
     end
 
     context "from 'yesterday'" do
       let(:options) { { from: 'yesterday' }}
-      its('time_range.from') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
     end
 
     context "from 'today'" do
       let(:options) { { from: 'today' }}
-      its('time_range.from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
     end
 
     context "from 'now'" do
       let(:options) { { from: 'now' }}
-      its('time_range.from') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
     end
 
     context "to 'yesterday'" do
       let(:options) { { to: 'yesterday' }}
-      its('time_range.to') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
+      its('to') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
     end
 
     context "to 'today'" do
       let(:options) { { to: 'today' }}
-      its('time_range.to') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+      its('to') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
     end
 
     context "to 'now'" do
       let(:options) { { to: 'now' }}
-      its('time_range.to') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+      its('to') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
     end
 
     context "from 'firstday'" do
 
       let(:options) { { from: 'firstday' } }
-      its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
 
       context "anchor 'yesterday'" do
         let(:anchor) { 'yesterday' }
-        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
       end
 
       context "anchor 'today'" do
         let(:anchor) { 'today' }
-        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
       end
 
       context "anchor '2014-07-17'" do
         let(:options) { { from: 'firstday', anchor: '2014-07-17' } }
-        its('time_range.from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
       end
 
       context "date range" do
         let(:options) { { from: 'firstday', anchor: 'yesterday' } }
-        its(:date_range) { should eq (Date.parse('2014-07-31')..Date.parse('2014-08-03')) }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
       end
 
-      context "indices" do
-        let(:options) { { from: 'firstday', anchor: 'yesterday' } }
-        its(:indices) {
-          should eq [
-            "logstash-2014.07.31",
-            "logstash-2014.08.01",
-            "logstash-2014.08.02",
-            "logstash-2014.08.03",
-          ]
-        }
+      context "each_period" do
+        let(:options) { { from: 'firstday', anchor: 'yesterday', to: 'today' } }
+        it "should iterate over range by hour" do
+          indices = []
+          queries = []
+          subject.each_period(3600) do |index, query|
+            indices << index
+            queries << query
+          end
+          expect(indices.count).to eq ((subject.to - subject.from)/3600).round
+          expect(indices.uniq).to eq %w(
+            logstash-2014.07.31
+            logstash-2014.08.01
+            logstash-2014.08.02
+          )
+          expect(queries.map(&:from).map(&:to_s)).to eq ([
+            "2014-07-31 22:00:00 UTC",
+            "2014-07-31 23:00:00 UTC",
+            "2014-08-01 00:00:00 UTC",
+            "2014-08-01 01:00:00 UTC",
+            "2014-08-01 02:00:00 UTC",
+            "2014-08-01 03:00:00 UTC",
+            "2014-08-01 04:00:00 UTC",
+            "2014-08-01 05:00:00 UTC",
+            "2014-08-01 06:00:00 UTC",
+            "2014-08-01 07:00:00 UTC",
+            "2014-08-01 08:00:00 UTC",
+            "2014-08-01 09:00:00 UTC",
+            "2014-08-01 10:00:00 UTC",
+            "2014-08-01 11:00:00 UTC",
+            "2014-08-01 12:00:00 UTC",
+            "2014-08-01 13:00:00 UTC",
+            "2014-08-01 14:00:00 UTC",
+            "2014-08-01 15:00:00 UTC",
+            "2014-08-01 16:00:00 UTC",
+            "2014-08-01 17:00:00 UTC",
+            "2014-08-01 18:00:00 UTC",
+            "2014-08-01 19:00:00 UTC",
+            "2014-08-01 20:00:00 UTC",
+            "2014-08-01 21:00:00 UTC",
+            "2014-08-01 22:00:00 UTC",
+            "2014-08-01 23:00:00 UTC",
+            "2014-08-02 00:00:00 UTC",
+            "2014-08-02 01:00:00 UTC",
+            "2014-08-02 02:00:00 UTC",
+            "2014-08-02 03:00:00 UTC",
+            "2014-08-02 04:00:00 UTC",
+            "2014-08-02 05:00:00 UTC",
+            "2014-08-02 06:00:00 UTC",
+            "2014-08-02 07:00:00 UTC",
+            "2014-08-02 08:00:00 UTC",
+            "2014-08-02 09:00:00 UTC",
+            "2014-08-02 10:00:00 UTC",
+            "2014-08-02 11:00:00 UTC",
+            "2014-08-02 12:00:00 UTC",
+            "2014-08-02 13:00:00 UTC",
+            "2014-08-02 14:00:00 UTC",
+            "2014-08-02 15:00:00 UTC",
+            "2014-08-02 16:00:00 UTC",
+            "2014-08-02 17:00:00 UTC",
+            "2014-08-02 18:00:00 UTC",
+            "2014-08-02 19:00:00 UTC",
+            "2014-08-02 20:00:00 UTC",
+            "2014-08-02 21:00:00 UTC"
+          ])
+        end
       end
 
     end
 
-    context "body" do
-      its(:body) { should eq ({
-        :sort => [{"@timestamp"=>{:order=>"asc"}}],
-        :fields => %w(message),
-        :query => {:filtered=>{
-          :query => { :bool => { :should => [ { :query_string => { :query=>"*" }}]}},
-          :filter=> { :bool => { :must   => [ { :range => { "@timestamp" => { :from => 1407016800000, :to => 1407074073000}}}]}}}}
-      })}
+    context "search" do
+      it "should produce the correct elasticsearch search request attributes" do
+        expect(subject.search(0, 10)).to eq ({
+          :sort => [{"@timestamp"=>{:order=>"asc"}}],
+          :fields => %w(message),
+          :query => {:filtered=>{
+            :query => { :bool => { :should => [ { :query_string => { :query=>"*" }}]}},
+            :filter=> { :bool => { :must   => [ { :range => { "@timestamp" => { :gte => 1407016800000, :lt => 1407074073000}}}]}}}},
+          :from => 0,
+          :size => 10
+        })
+      end
     end
 
   end
 
   context "running on 2014-08-01" do
     let(:time)  { '2014-08-01 12:53:03' }
-    let(:query) { nil }
+    let(:query_string) { nil }
     let(:options) { {} }
 
-    subject { Lstash::Query.new(query, options) }
+    subject { Lstash::Query.new(query_string, options) }
 
     before { Timecop.freeze(Time.parse(time)) }
     after  { Timecop.return }
@@ -129,15 +190,15 @@ describe Lstash::Query do
     context "from 'firstday' with 'yesterday' anchor" do
       let(:options) { { anchor: 'yesterday', from: 'firstday' } }
 
-        its('time_range.from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
-        its('time_range.to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
+        its('to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
     end
 
     context "from 'firstday' with default 'today' anchor" do
       let(:options) { { from: 'firstday', to: 'now' } }
 
-        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
-        its('time_range.to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
     end
 
   end

metadata

@@ -1,183 +1,155 @@
 --- !ruby/object:Gem::Specification
 name: lstash
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.2.0
 platform: ruby
 authors:
 - Klaas Jan Wierenga
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-09-22 00:00:00.000000000 Z
+date: 2021-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 10.3.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 10.3.2
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: rspec-its
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
-  name: rspec-autotest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: autotest-standalone
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: autotest-fsevent
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.1
 - !ruby/object:Gem::Dependency
-  name: timecop
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 1.4.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: elasticsearch
+  name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 4.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 4.1.0
 - !ruby/object:Gem::Dependency
-  name: hashie
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.20.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.20.3
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.17.4
 description: Count or grep log messages in a specified time range from a Logstash
   Elasticsearch server.
 email:
@@ -190,12 +162,12 @@ extra_rdoc_files:
 - README.md
 - CHANGELOG.md
 files:
-- .autotest
-- .gitignore
-- .rspec
-- .ruby-gemset
-- .ruby-version
-- .travis.yml
+- ".autotest"
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -213,28 +185,27 @@ files:
 - spec/lstash/query_spec.rb
 - spec/lstash_spec.rb
 - spec/spec_helper.rb
-homepage: http://bitbucket.org/kjwierenga/lstash
+homepage: https://github.com/kjwierenga/lstash
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.1
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: The lstash gem allows you to count or grep log messages in a specific time
   range from a Logstash Elasticsearch server.