lstash 0.0.9 → 0.1.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NTgxYmI5NWQ2ZmE3OGY1NDE0ZGNjN2M1ZGQ2ZWYxNGI5ODY0MGY1NQ==
+    N2RmOWQ3YWIxZjRiZTk5MTc3ZTI5MGJlMDEwZTNmNThjMmY3OTJkMg==
   data.tar.gz: !binary |-
-    ZTgzZmQ0YTU3MTAwZDA1MWI1MjNmMjM0M2M5NWY1Mjk5NWRmMTM4OA==
+    NTUzMzJkZGI4NmEyZDUwZGRkZWI0YTI2OTlhNDU3ODdkNmQwOWM5NA==
 SHA512:
   metadata.gz: !binary |-
-    YThlNDM0OTNjZTNhYWI3ODc4MDc5ZjUyOGJhMTBmZTc4YjI3Nzc2YTZiMjFl
-    ZDk0YTQ3NGNkZDNmMThhN2Y2ODVmOWY5NTliYzQ3MTZmNzc2YzEzZTNmZjU4
-    YjAzMTMxNzEwZDgzNDdjMTM2ZTRlYjZjNjZmM2VlMzJkMTE0NDY=
+    YWJhYjJjM2Y5ZDAzYmE3ODY5ZTYxNTMyNGY4ODZiN2M4MTg3YmI2MDk5YmJm
+    NTY0MWJhMzBlOTU2MWUwZWM5NWU3MDllNTk3MTMwNzg3ODlhMjI1ZmNmYTc1
+    ZWI3YzBkZTg1YzA3MDg0YWNjNWJjZmQyZDMxMDc1NjY5YTBhMWM=
   data.tar.gz: !binary |-
-    N2Y5NWI5ZDdiNGNjOTFhMTJkMjJjMzVmZmExZTQ1NjFhZjMzNGU3MDdjMDFi
-    ZDE4Y2U5MGUyYzY3NTI5N2MyMDE5MDY2NjVhYWVhZTY2Mzk5YmFmNmVkM2Rm
-    ZjI2ZmYwOTE5NWMxNmI5YTlmYTdjMjRkY2QxZDRlOWNmMjAzNDU=
+    MWUyMzNiMjU3ODM0ZTYzZWM3OGM3OGUyMDM3OGEyMDFiMTE4MjNiOWIzNmU4
+    ZDYzMDI0MzlhMGJmMTVhNWQ4OGU5ODcyZjQ4ZDE2NWJkYTVhMGM2NmEzOGQ4
+    NjhkOGVhYWEwNjcxZjU5OTMzNjcxMjRiOGIzNmUzODg4OWRiYzE=

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+### 0.1.1 / 2014-10-13
+
+Enhancements
+
+* Refactor to iterate over hour periods instead of full indices. This improves the performance a lot.
+* Increase timeout to prevent Faraday::TimeoutError exceptions.
+
 ### 0.0.9 / 2014-09-22
 
 Enhancements

data/lib/lstash/cli.rb

@@ -9,6 +9,8 @@ require 'lstash/client'
 
 module Lstash
 
+  TRANSPORT_REQUEST_TIMEOUT = 60.freeze # 1 minute request timeout
+
   class CLI < Thor
 
     class_option :from,   :banner => 'start of time range', :aliases => '-f', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
@@ -45,7 +47,7 @@ module Lstash
     desc "count QUERY", "count number of log messages matching the QUERY"
     def count(query_string)
       run_command(query_string) do |es_client, query|
-        count  = Lstash::Client.new(es_client, options).count(query)
+        count = Lstash::Client.new(es_client, options).count(query)
         puts count
       end
     end
@@ -55,14 +57,15 @@ module Lstash
     def run_command(query_string)
       es_client = ::Elasticsearch::Client.new(
         url: options[:es_url] || ENV['ES_URL'] || 'localhost',
-        log: !!ENV['DEBUG']
+        log: !!ENV['DEBUG'],
+        transport_options: { request: { timeout: TRANSPORT_REQUEST_TIMEOUT } }
       )
       query  = Lstash::Query.new(query_string, options)
 
       yield es_client, query
 
     rescue Exception => e
-      raise Thor::Error.new(e.message)
+      options[:debug] ? raise(e) : raise(Thor::Error.new(e.message))
     end
 
     protected

data/lib/lstash/client.rb

@@ -13,7 +13,7 @@ module Lstash
 
     class ConnectionError < StandardError; end
 
-    PER_PAGE = 5000 # best time, lowest resource usage
+    PER_PAGE = 5000.freeze # best time, lowest resource usage
 
     def initialize(es_client, options = {})
       raise ConnectionError, "No elasticsearch client specified" if es_client.nil?
@@ -23,24 +23,29 @@ module Lstash
     end
 
     def count(query)
-      @logger.debug "time range: [%s..%s]" % [query.time_range.from, query.time_range.to]
+      @logger.debug "count from=#{query.from} to=#{query.to}"
 
       count = 0
-      query.indices.each do |index|
-        count += count_messages(index, query)
+      query.each_hour do |index, hour_query|
+        count += count_messages(index, hour_query)
       end
-
+      @logger.debug "total count=#{count}"
       count
     end
 
     def grep(query)
-      @logger.debug "time range: [%s..%s]" % [query.time_range.from, query.time_range.to]
+      @logger.debug "grep from=#{query.from} to=#{query.to}"
 
-      query.indices.each do |index|
-        grep_messages(index, query) do |message|
+      count = 0
+      query.each_hour do |index, hour_query|
+        grep_messages(index, hour_query) do |message|
+          count += 1
           yield message if block_given?
         end
       end
+
+      @logger.debug "total count=#{count}"
+      count
     end
 
     private
@@ -48,9 +53,9 @@ module Lstash
     def count_messages(index, query)
       result = Hashie::Mash.new @es_client.send(:count,
         index: index,
-        body:  query.body[:query]
+        body:  query.filter
       )
-      @logger.debug "count #{index}: #{result['count']} "
+      @logger.debug "count index=#{index} from=#{query.from} to=#{query.to} count=#{result['count']}"
       result['count']
     end
 
@@ -61,9 +66,9 @@ module Lstash
       method = :search
       while (messages.nil? || messages.count > 0) do
         result = Hashie::Mash.new @es_client.send(method, {
-          index: index,
-          scroll: '10m',
-          body: query.body.merge(from: offset, size: PER_PAGE),
+          index:  index,
+          scroll: '1m',
+          body:   query.search(offset, PER_PAGE),
         }.merge(scroll_params))
 
         messages = result.hits.hits
@@ -72,19 +77,20 @@ module Lstash
         scroll_params = {scroll_id: result._scroll_id}
 
         messages.each do |h|
+          next if h.fields.nil?
           yield h.fields.message if block_given?
         end
 
         method = :scroll
       end
-      @logger.debug "grep #{index}: #{offset}"
+      @logger.debug "grep index=#{index} from=#{query.from} to=#{query.to} count=#{offset}"
       Hashie::Mash.new @es_client.clear_scroll(scroll_params)
     end
 
     def debug_logger
       logger = Logger.new(STDERR)
       logger.formatter = proc do |severity, datetime, progname, msg|
-        "#{msg}\n"
+        "#{datetime} #{msg}\n"
       end
       logger
     end

data/lib/lstash/query.rb

@@ -9,11 +9,14 @@ module Lstash
     class FormatError < StandardError; end
     class QueryMissing < StandardError; end
 
-    LOGSTASH_PREFIX = 'logstash-'
-    WILDCARD_QUERY = '*'
+    LOGSTASH_PREFIX = 'logstash-'.freeze
+    WILDCARD_QUERY  = '*'.freeze
+    HOUR_IN_SECONDS = 3600.freeze
 
-    def initialize(query = nil, arguments = {})
-      @query = query
+    attr_accessor :from, :to
+
+    def initialize(query_string = nil, arguments = {})
+      @query_string = query_string
 
       @anchor = time_parse(arguments[:anchor], 'today')
       @from   = time_parse(arguments[:from],   'today')
@@ -22,38 +25,51 @@ module Lstash
       @to = Time.now if @to > Time.now # prevent accessing non-existing times / indices
     end
 
-    def time_range
-      OpenStruct.new(from: @from, to: @to)
-    end
-
-    def date_range
-      (@from.utc.to_date .. @to.utc.to_date)
+    def index_name(date)
+      "#{LOGSTASH_PREFIX}#{date.strftime('%Y.%m.%d')}"
     end
 
-    def indices
-      date_range.map { |d| "#{LOGSTASH_PREFIX}#{d.strftime('%Y.%m.%d')}" }
-    end
-
-    def body
+    def search(from, size)
       {
-        sort: sort_order,
-
+        sort:   sort_order,
         fields: %w(message),
+        query:  filter,
+        from:   from,
+        size:   size
+      }
+    end
 
-        # return in order of ascending timestamp
-        query: {
-          filtered: {
-            query:  es_query,
-            filter: es_filter
-          }
+    def filter
+      {
+        filtered: {
+          query:  es_query,
+          filter: es_filter
         }
       }
     end
 
+    def each_hour(&block)
+      # iterate over the whole range in blocks of one hour
+      time_iterate(@from.utc, @to.utc - 1, HOUR_IN_SECONDS) do |hour|
+        yield index_name(hour.to_date),
+              Query.new(@query_string,
+                        anchor: @anchor,
+                        from:   hour,
+                        to:     hour + HOUR_IN_SECONDS)
+      end
+    end
+
     private
 
-    def time_parse(time_string, default)
-      time_string = time_string.strip rescue nil
+    def time_iterate(start_time, end_time, step, &block)
+      begin
+        yield(start_time)
+      end while (start_time += step) < end_time
+    end
+
+    def time_parse(time_or_string, default)
+      return time_or_string if time_or_string.is_a? Time
+      time_string = time_or_string.strip rescue nil
       time_string ||= default
       case time_string
       when 'firstday'
@@ -71,13 +87,14 @@ module Lstash
       raise FormatError, "Invalid time format: #{time_string}"
     end
 
-    def query
-      q = @query.dup.strip rescue ''
+    def query_string
+      q = @query_string.dup.strip rescue ''
       q = WILDCARD_QUERY if q.empty?
       q
     end
 
     def sort_order
+      # return results in order of ascending timestamp
       [ { '@timestamp' => { order: 'asc' } } ]
     end
 
@@ -87,7 +104,7 @@ module Lstash
           should: [
             {
               query_string: {
-                query: query
+                query: query_string
               }
             }
           ]
@@ -101,14 +118,14 @@ module Lstash
           must: [
             range: {
               '@timestamp' => {
-                from: to_msec(time_range.from),
-                to:   to_msec(time_range.to)
+                gte: to_msec(from),
+                lt:  to_msec(to)
               }
             },
             # fquery: {
             #   query: {
             #     query_string: {
-            #       query: query
+            #       query: query_string
             #     }
             #   }
             # }
@@ -117,7 +134,7 @@ module Lstash
           #   fquery: {
           #     query: {
           #       query_string: {
-          #         query: query
+          #         query: query_string
           #       }
           #     }
           #   }
@@ -126,7 +143,7 @@ module Lstash
           #   fquery: {
           #     query: {
           #       query_string: {
-          #         query: query
+          #         query: query_string
           #       }
           #     }
           #   }

data/lib/lstash/version.rb

@@ -1,3 +1,3 @@
 module Lstash
-  VERSION = "0.0.9"
+  VERSION = "0.1.1"
 end

data/spec/lstash/cli_spec.rb

@@ -9,6 +9,13 @@ end
 
 describe Lstash::CLI do
 
+  before(:all) do
+    Timecop.freeze('2014-08-01 14:58')
+  end
+  after(:all) do
+    Timecop.return
+  end
+
   context "options" do
     subject { Lstash::CLI.options(args) }
 
@@ -25,9 +32,12 @@ describe Lstash::CLI do
         client = double('client')
 
         allow(Lstash::Client).to receive(:new).and_return(client)
-        allow(client).to receive(:count).and_return(100)
+        allow(client).to receive(:count).and_return(1000)
 
-        expect { Lstash::CLI.start(args) }.not_to raise_error
+        expect {
+          output = capture_stdout { Lstash::CLI.start(args) }
+          expect(output).to eq "1000\n"
+        }.not_to raise_error
       end
     end
 
@@ -68,52 +78,33 @@ describe Lstash::CLI do
     context "with anchor date" do
       let(:args) { %w(count program:haproxy --from firstday --to today --anchor yesterday) }
 
-      it "should succeed" do
-        Timecop.freeze('2014-08-01 14:58') do
-          es_client = double('es_client')
+      it "should return correct count" do
+        es_client = double('es_client')
 
-          allow(Elasticsearch::Client).to receive(:new) { es_client }
+        allow(Elasticsearch::Client).to receive(:new) { es_client }
 
-          expect(es_client).to receive(:count).with(satisfy { |args|
-            expect_time_range(args, [
-              Time.parse('2014-07-01').to_i*1000,
-              Time.parse('2014-08-01').to_i*1000
-            ])
-          }).exactly(32).times.and_return({count:1})
+        expect(es_client).to receive(:count).exactly(31 * 24).times.and_return(count:100)
 
-          Lstash::CLI.start(args)
-        end
+        output = capture_stdout { Lstash::CLI.start(args) }
+        expect(output).to match("#{31 * 24 * 100}")
       end
     end
 
     context "without anchor date" do
       let(:args) { %w(count program:haproxy --from yesterday --to today) }
 
-      it "should succeed" do
-        Timecop.freeze('2014-08-01 14:58') do
-          es_client = double('es_client')
+      it "should return correct count" do
+        es_client = double('es_client')
 
-          allow(Elasticsearch::Client).to receive(:new) { es_client }
+        allow(Elasticsearch::Client).to receive(:new) { es_client }
 
-          expect(es_client).to receive(:count).with(satisfy { |args|
-            expect_time_range(args, [
-              Time.parse('2014-07-31').to_i*1000,
-              Time.parse('2014-08-01').to_i*1000
-            ])
-          }).exactly(2).times.and_return({count:1})
+        expect(es_client).to receive(:count).exactly(24).times.and_return(count:100)
 
-          Lstash::CLI.start(args)
-        end
+        output = capture_stdout { Lstash::CLI.start(args) }
+        expect(output).to match("#{24 * 100}")
       end
     end
 
   end
 
-  private
-
-  def expect_time_range(args, time_range)
-    expect(args[:body][:filtered][:filter][:bool][:must].first[:range]['@timestamp'][:from]).to eq time_range.first
-    expect(args[:body][:filtered][:filter][:bool][:must].first[:range]['@timestamp'][:to]).to   eq time_range.last
-  end
-
 end

data/spec/lstash/client_spec.rb

@@ -12,38 +12,45 @@ describe Lstash::Client do
 
   context "with query" do
 
-    let(:query) { double('query', time_range: OpenStruct.new) }
-
     context "count" do
       it "should return number of messages matching query" do
-        allow(query).to receive(:indices).and_return (['logstash-2014-08-01', 'logstash-2014-08-02'])
-        allow(query).to receive(:body).and_return ({})
 
-        allow(es_client).to receive(:count).and_return({'count' => 100},{'count' => 100})
+        query = Lstash::Query.new("*", from: Time.parse("2014-10-10 00:00"), to: Time.parse("2014-10-10 07:00"))
+
+        allow(es_client).to receive(:count).and_return(
+          { 'count' => 100 },
+          { 'count' => 200 },
+          { 'count' => 300 },
+          { 'count' => 400 },
+          { 'count' => 500 },
+          { 'count' => 600 },
+          { 'count' => 700 }
+        )
 
-        expect(subject.count(query)).to eq 200
+        expect(subject.count(query)).to eq 2800
       end
     end
 
     context "grep" do
-      let(:query) { double('query', time_range: OpenStruct.new) }
-      
       it "should return the messages matching the query" do
-        allow(query).to receive(:indices).and_return (['logstash-2014-08-01', 'logstash-2014-08-02'])
-        allow(query).to receive(:body).and_return ({})
-        
-        allow(es_client).to receive(:search).and_return(
-          hits([
-            'this is the first log line',
-            'this is the second log line'
-          ])
+
+        query = Lstash::Query.new("*", from: Time.parse("2014-10-10 00:00"), to: Time.parse("2014-10-10 07:00"))
+      
+        expect(es_client).to receive(:search).and_return(
+          hits(%w(1)),
+          hits(%w(2 2)),
+          hits(%w(3 3 3)),
+          hits(%w(4 4 4 4)),
+          hits(%w(5 5 5 5 5)),
+          hits(%w(6 6 6 6 6 6)),
+          hits(%w(7 7 7 7 7 7 7))
         )
 
         allow(es_client).to receive(:scroll).and_return(hits([]))
 
         allow(es_client).to receive(:clear_scroll)
 
-        subject.grep(query)
+        expect(subject.grep(query)).to eq 28
       end
     end
 

data/spec/lstash/query_spec.rb

@@ -5,10 +5,10 @@ describe Lstash::Query do
 
   context "running on 2014-08-03" do
     let(:time)  { '2014-08-03 15:54:33' }
-    let(:query) { nil }
+    let(:query_string) { nil }
     let(:options) { {} }
 
-    subject { Lstash::Query.new(query, options) }
+    subject { Lstash::Query.new(query_string, options) }
 
     before { Timecop.freeze(Time.parse(time)) }
     after  { Timecop.return }
@@ -18,110 +18,171 @@ describe Lstash::Query do
     #   expect(subject).not_to be nil
     # end
 
-    its('time_range.from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
-    its('time_range.to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+    its('from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+    its('to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
 
-    its(:date_range) { should eq (Date.parse('2014-08-02')..Date.parse('2014-08-03')) }
+    # its(:date_range) { should eq (Date.parse('2014-08-02')..Date.parse('2014-08-03')) }
 
-    its(:indices) { should eq [ "logstash-2014.08.02", "logstash-2014.08.03" ] }
+    # its(:indices) { should eq [ "logstash-2014.08.02", "logstash-2014.08.03" ] }
 
     context "with specific query" do
-      let(:query) { 'program:haproxy' }
-      its(:query) { should eq query }
+      let(:query_string) { 'program:haproxy' }
+      its(:query_string) { should eq query_string }
     end
 
     context "without query" do
-      let(:query) { nil }
-      its(:query) { should eql '*' }
+      let(:query_string) { nil }
+      its(:query_string) { should eql '*' }
     end
 
     context "from 'yesterday'" do
       let(:options) { { from: 'yesterday' }}
-      its('time_range.from') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
     end
 
     context "from 'today'" do
       let(:options) { { from: 'today' }}
-      its('time_range.from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
     end
 
     context "from 'now'" do
       let(:options) { { from: 'now' }}
-      its('time_range.from') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
     end
 
     context "to 'yesterday'" do
       let(:options) { { to: 'yesterday' }}
-      its('time_range.to') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
+      its('to') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
     end
 
     context "to 'today'" do
       let(:options) { { to: 'today' }}
-      its('time_range.to') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+      its('to') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
     end
 
     context "to 'now'" do
       let(:options) { { to: 'now' }}
-      its('time_range.to') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+      its('to') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
     end
 
     context "from 'firstday'" do
 
       let(:options) { { from: 'firstday' } }
-      its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+      its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
 
       context "anchor 'yesterday'" do
         let(:anchor) { 'yesterday' }
-        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
       end
 
       context "anchor 'today'" do
         let(:anchor) { 'today' }
-        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
       end
 
       context "anchor '2014-07-17'" do
         let(:options) { { from: 'firstday', anchor: '2014-07-17' } }
-        its('time_range.from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
       end
 
       context "date range" do
         let(:options) { { from: 'firstday', anchor: 'yesterday' } }
-        its(:date_range) { should eq (Date.parse('2014-07-31')..Date.parse('2014-08-03')) }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
       end
 
-      context "indices" do
-        let(:options) { { from: 'firstday', anchor: 'yesterday' } }
-        its(:indices) {
-          should eq [
-            "logstash-2014.07.31",
-            "logstash-2014.08.01",
-            "logstash-2014.08.02",
-            "logstash-2014.08.03",
-          ]
-        }
+      context "each_hour" do
+        let(:options) { { from: 'firstday', anchor: 'yesterday', to: 'today' } }
+        it "should iterate over range by hour" do
+          indices = []
+          queries = []
+          subject.each_hour do |index, query|
+            indices << index
+            queries << query
+          end
+          expect(indices.count).to eq ((subject.to - subject.from)/3600).round
+          expect(indices.uniq).to eq %w(
+            logstash-2014.07.31
+            logstash-2014.08.01
+            logstash-2014.08.02
+          )
+          expect(queries.map(&:from).map(&:to_s)).to eq ([
+            "2014-07-31 22:00:00 UTC",
+            "2014-07-31 23:00:00 UTC",
+            "2014-08-01 00:00:00 UTC",
+            "2014-08-01 01:00:00 UTC",
+            "2014-08-01 02:00:00 UTC",
+            "2014-08-01 03:00:00 UTC",
+            "2014-08-01 04:00:00 UTC",
+            "2014-08-01 05:00:00 UTC",
+            "2014-08-01 06:00:00 UTC",
+            "2014-08-01 07:00:00 UTC",
+            "2014-08-01 08:00:00 UTC",
+            "2014-08-01 09:00:00 UTC",
+            "2014-08-01 10:00:00 UTC",
+            "2014-08-01 11:00:00 UTC",
+            "2014-08-01 12:00:00 UTC",
+            "2014-08-01 13:00:00 UTC",
+            "2014-08-01 14:00:00 UTC",
+            "2014-08-01 15:00:00 UTC",
+            "2014-08-01 16:00:00 UTC",
+            "2014-08-01 17:00:00 UTC",
+            "2014-08-01 18:00:00 UTC",
+            "2014-08-01 19:00:00 UTC",
+            "2014-08-01 20:00:00 UTC",
+            "2014-08-01 21:00:00 UTC",
+            "2014-08-01 22:00:00 UTC",
+            "2014-08-01 23:00:00 UTC",
+            "2014-08-02 00:00:00 UTC",
+            "2014-08-02 01:00:00 UTC",
+            "2014-08-02 02:00:00 UTC",
+            "2014-08-02 03:00:00 UTC",
+            "2014-08-02 04:00:00 UTC",
+            "2014-08-02 05:00:00 UTC",
+            "2014-08-02 06:00:00 UTC",
+            "2014-08-02 07:00:00 UTC",
+            "2014-08-02 08:00:00 UTC",
+            "2014-08-02 09:00:00 UTC",
+            "2014-08-02 10:00:00 UTC",
+            "2014-08-02 11:00:00 UTC",
+            "2014-08-02 12:00:00 UTC",
+            "2014-08-02 13:00:00 UTC",
+            "2014-08-02 14:00:00 UTC",
+            "2014-08-02 15:00:00 UTC",
+            "2014-08-02 16:00:00 UTC",
+            "2014-08-02 17:00:00 UTC",
+            "2014-08-02 18:00:00 UTC",
+            "2014-08-02 19:00:00 UTC",
+            "2014-08-02 20:00:00 UTC",
+            "2014-08-02 21:00:00 UTC"
+          ])
+        end
       end
 
     end
 
-    context "body" do
-      its(:body) { should eq ({
-        :sort => [{"@timestamp"=>{:order=>"asc"}}],
-        :fields => %w(message),
-        :query => {:filtered=>{
-          :query => { :bool => { :should => [ { :query_string => { :query=>"*" }}]}},
-          :filter=> { :bool => { :must   => [ { :range => { "@timestamp" => { :from => 1407016800000, :to => 1407074073000}}}]}}}}
-      })}
+    context "search" do
+      it "should produce the correct elasticsearch search request attributes" do
+        expect(subject.search(0, 10)).to eq ({
+          :sort => [{"@timestamp"=>{:order=>"asc"}}],
+          :fields => %w(message),
+          :query => {:filtered=>{
+            :query => { :bool => { :should => [ { :query_string => { :query=>"*" }}]}},
+            :filter=> { :bool => { :must   => [ { :range => { "@timestamp" => { :gte => 1407016800000, :lt => 1407074073000}}}]}}}},
+          :from => 0,
+          :size => 10
+        })
+      end
     end
 
   end
 
   context "running on 2014-08-01" do
     let(:time)  { '2014-08-01 12:53:03' }
-    let(:query) { nil }
+    let(:query_string) { nil }
     let(:options) { {} }
 
-    subject { Lstash::Query.new(query, options) }
+    subject { Lstash::Query.new(query_string, options) }
 
     before { Timecop.freeze(Time.parse(time)) }
     after  { Timecop.return }
@@ -129,15 +190,15 @@ describe Lstash::Query do
     context "from 'firstday' with 'yesterday' anchor" do
       let(:options) { { anchor: 'yesterday', from: 'firstday' } }
 
-        its('time_range.from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
-        its('time_range.to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
+        its('to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
     end
 
     context "from 'firstday' with default 'today' anchor" do
       let(:options) { { from: 'firstday', to: 'now' } }
 
-        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
-        its('time_range.to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
+        its('from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lstash
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.1
 platform: ruby
 authors:
 - Klaas Jan Wierenga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-22 00:00:00.000000000 Z
+date: 2014-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -233,7 +233,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: The lstash gem allows you to count or grep log messages in a specific time