lstash 0.1.3 → 0.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZTI2NWNjODBjNjBmNjEyOTE2OGMwMmEyMDdjYjFhZDAzYjU0ODZmYQ==
+    Y2YxYjRjNGYyMTA4MTdiNjQ5NjA2ZDU2MmE4Y2Y3MmZkNDRjYWMzYg==
   data.tar.gz: !binary |-
-    NmFlZDNlYzU5OGJiYjk2MWU0MzZiNWJlZjM2MWQwZmNmMzUyNGQ0Mg==
+    Y2NhOTJlMmRjZmZlZmNkZGRhMDZkYmRhZWMwMzY5M2U4YTM0ZGE1YQ==
 SHA512:
   metadata.gz: !binary |-
-    YTAyNTIwYzBmZGM1MjYwYzgxMjFhZjAwMjFlYzU0OTMwMjM3M2UwZDAyOGIy
-    YTdmOWFiM2I2YzFjMjI2YmZiODczMzQ0OTA0ZTdjYzU4NDYwYzcwMjJkODQz
-    N2NkZGIwZDU1MjA1NGQ4ODE2NzgyOTIxY2MyMWNjMmMyYzZkZjg=
+    MjA3MzA0OWQ3ZTVmOWU4YmE0NjdmNWQ4NjgzMTA0NTg4ZTcxNmJlZmI2YTA5
+    OGQ2Yzk5NTk1NjNhYzFmOTA3NDk0NmYyNjlmNzA4M2M2MzdiNzE4MmNiNDEz
+    MGZlNTg1Y2UxN2U5MzAxZmEyM2YxZmM1MDAwOTYwYzRjMWFiZDQ=
   data.tar.gz: !binary |-
-    YWQ1ZDA1ZWFiODU4OGUzZWY2MWE2ZmQwNDBiOThlODNmNmQyYzQ3ODlkNzVh
-    NzA4Njg0N2FhMTU5NGI3OGJlODc5YjlhNzg2MzcwZTFlMGFkNzQyOGZhOGQx
-    NmVmNzZmNDAyOGU5YTBmN2IwMGZmMjFkMWYwNzQ2ZTlkODA5NjE=
+    YTQ5MmJiY2E4NjlkYzRlNTcyYWM0ZjA3NDQyNTc5MzNhNjhmMzVhM2MyOTZi
+    OGFjMzQ1Yjk1OTgzNDE1YmJhMzBmNDI1YmYxMDhjZmJlNmJkZjBlN2U2YTI5
+    ZGRlYWI0NjMwY2I0YjZlN2RlOWM3NDYxNTg5MjljZTlhNThiNDc=

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.log

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+### 0.1.4 / 2015-05-29
+
+Enhancements
+
+* Change default time steps from 1 hour to 120 seconds to aid in grepping from datasets with many events per minute.
+
+### 0.1.3 / 2015-05-28
+
+Enhancements
+
+* Remove troublesome development dependencies (autotest).
+
 ### 0.1.2 / 2015-01-02
 
 Enhancements

data/lib/lstash/client.rb

@@ -14,6 +14,8 @@ module Lstash
     class ConnectionError < StandardError; end
 
     PER_PAGE = 5000.freeze # best time, lowest resource usage
+    DEFAULT_COUNT_STEP = 3600.freeze # 1 hour
+    DEFAULT_GREP_STEP  = 120.freeze  # 2 minutes
 
     def initialize(es_client, options = {})
       raise ConnectionError, "No elasticsearch client specified" if es_client.nil?
@@ -22,22 +24,22 @@ module Lstash
       @logger    = options[:logger] || (options[:debug] ? debug_logger : NullLogger.new)
     end
 
-    def count(query)
+    def count(query, step = DEFAULT_COUNT_STEP)
       @logger.debug "count from=#{query.from} to=#{query.to}"
 
       count = 0
-      query.each_hour do |index, hour_query|
+      query.each_period(step) do |index, hour_query|
         count += count_messages(index, hour_query)
       end
       @logger.debug "total count=#{count}"
       count
     end
 
-    def grep(query)
+    def grep(query, step = DEFAULT_GREP_STEP)
       @logger.debug "grep from=#{query.from} to=#{query.to}"
 
       count = 0
-      query.each_hour do |index, hour_query|
+      query.each_period(step) do |index, hour_query|
         grep_messages(index, hour_query) do |message|
           count += 1
           yield message if block_given?
@@ -67,7 +69,7 @@ module Lstash
       while (messages.nil? || messages.count > 0) do
         result = Hashie::Mash.new @es_client.send(method, {
           index:  index,
-          scroll: '1m',
+          scroll: '5m',
           body:   query.search(offset, PER_PAGE),
         }.merge(scroll_params))
 

data/lib/lstash/query.rb

@@ -11,7 +11,6 @@ module Lstash
 
     LOGSTASH_PREFIX = 'logstash-'.freeze
     WILDCARD_QUERY  = '*'.freeze
-    HOUR_IN_SECONDS = 3600.freeze
 
     attr_accessor :from, :to
 
@@ -48,14 +47,14 @@ module Lstash
       }
     end
 
-    def each_hour(&block)
-      # iterate over the whole range in blocks of one hour
-      time_iterate(@from.utc, @to.utc - 1, HOUR_IN_SECONDS) do |hour|
-        yield index_name(hour.to_date),
+    def each_period(step, &block)
+      # iterate over the whole range in blocks of the specified period
+      time_iterate(@from.utc, @to.utc - 1, step) do |start_at|
+        yield index_name(start_at.to_date),
               Query.new(@query_string,
                         anchor: @anchor,
-                        from:   hour,
-                        to:     hour + HOUR_IN_SECONDS)
+                        from:   start_at,
+                        to:     start_at + step)
       end
     end
 

data/lib/lstash/version.rb

@@ -1,3 +1,3 @@
 module Lstash
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/spec/lstash/client_spec.rb

@@ -50,7 +50,7 @@ describe Lstash::Client do
 
         allow(es_client).to receive(:clear_scroll)
 
-        expect(subject.grep(query)).to eq 28
+        expect(subject.grep(query, 3600)).to eq 28
       end
     end
 

data/spec/lstash/query_spec.rb

@@ -91,12 +91,12 @@ describe Lstash::Query do
         its('to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
       end
 
-      context "each_hour" do
+      context "each_period" do
         let(:options) { { from: 'firstday', anchor: 'yesterday', to: 'today' } }
         it "should iterate over range by hour" do
           indices = []
           queries = []
-          subject.each_hour do |index, query|
+          subject.each_period(3600) do |index, query|
             indices << index
             queries << query
           end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lstash
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Klaas Jan Wierenga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-28 00:00:00.000000000 Z
+date: 2015-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler