lstash 0.0.6

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MWVmNzk3MjRlYTIxNjRkMzNiYjliODg1ZTEzZmNkOTJlMDE4ZDA5Ng==
+  data.tar.gz: !binary |-
+    ZDRiNGE0ZmZhZTViOGI0YjFmNjU3ZTJjYTAxNWY1Y2Y0MTdhODZkZg==
+SHA512:
+  metadata.gz: !binary |-
+    N2QyNTU3ZTBiNDRiZDgxMTViNWRhM2RjNTZkZTM5ZmI0M2NiNjkyZTE3YTEy
+    ZjJlNmIxM2YzMTVlM2MxY2JmNDMwNTgyZjcxODRiMGNiNGE0YzI0ZDk5Yjkw
+    NmRlMDA3YWUyOGRlYmNhMjc5NTNlYWEwNDIxOGEyZGVjMzFhNTc=
+  data.tar.gz: !binary |-
+    NTU0ZDk2MTkyMzk4ODgzZjk1YzI3YTIyYTQ5ZmExMGFiYWIxY2JjZWVjNDI0
+    ZjcwMTZlNWMxOTY1NTcyMDhiNWQ2NzhiYmI0YWRjMGY3MTgxM2YwMjZmNDE2
+    ZGI3NDFmMTM4OWI5MmM1ZWZmMGNhZTJkOTg1NDFlMzg5NzYyOTY=

data/.autotest

@@ -0,0 +1,2 @@
+require 'autotest/bundler'
+require 'autotest/fsevent'

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+lstash

data/.ruby-version

@@ -0,0 +1 @@
+ruby-1.9.3

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+  - jruby-19mode

data/CHANGELOG.md

@@ -0,0 +1,37 @@
+### v0.0.6 / 2014-09-01
+
+Bug Fixes
+
+* Pushing version 0.0.5 failed and it was yanked from rubygems.org. Need to push new version 0.0.6.
+
+### 0.0.5 / 2014-09-01
+
+Bug Fixes
+
+* Running as binary didn't load HTTP client properly. Add 'patron' dependency
+  to force loading an appropriate HTTP client.
+
+### 0.0.4 / 2014-08-29
+
+Bug Fixes
+
+* Use .ruby-[version|gemset] to support any ruby environment manager.
+
+### 0.0.3 / 2014-08-28
+
+Bug Fixes
+
+* Run CI on travis-ci.org.
+* Fixate timezone to assumed timezone for specs.
+* Run Ruby 2.1.1 instead of 2.1.0.
+
+### 0.0.2 / 2014-08-28
+
+Enhancements
+
+* Updated documentation.
+* Rename debug option (-v) to (-d).
+
+### 0.0.1 / 2014-08-28
+
+Initial release.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in lstash.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Klaas Jan Wierenga
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,154 @@
+# lstash
+
+[![Build Status](https://travis-ci.org/kjwierenga/lstash.svg?branch=master)](https://travis-ci.org/kjwierenga/lstash)
+
+Lstash is a gem and command line utility to count or grep log messages in a certain time frame from a Logstash Elasticsearch server.
+
+## Installation
+
+Or install it yourself as:
+
+    $ gem install lstash
+
+## Running lstash from the command line
+
+	$ lstash
+	Commands:
+	  lstash count QUERY     # count number of log messages matching the QUERY
+	  lstash grep QUERY      # grep log messages from Logstash
+	  lstash help [COMMAND]  # Describe available commands or one specific command
+
+## The `count` command
+
+	Usage:
+	  lstash count QUERY
+
+	Description:
+	  Count log messages matching the QUERY from Logstash and output this count to stdout. QUERY can use Apache Lucene query 
+	  parser syntax.
+
+	  Example to count the number of HAProxy log messages in yesterdays month.
+
+	  lstash count 'program:haproxy' --from firstday --to today --anchor yesterday
+
+## The `grep` command
+
+	Usage:
+	  lstash grep QUERY
+
+	Description:
+	  Grep log messages matching the QUERY from Logstash in ascending timestamp order and output to stdout. QUERY can use Apache Lucene query parser syntax.
+
+	  Example to grep HAProxy log messages from the beginning of this month upto now
+
+	  lstash grep 'program:haproxy' --from firstday --to now
+
+## Command line options
+
+	Options:
+	  -f, [--from=start of time range]                    # date/time, 'now', 'today', 'yesterday', or 'firstday'
+	  -t, [--to=end of time range]                        # date/time, 'now', 'today', 'yesterday', or 'firstday'
+	  -a, [--anchor=anchor date/time]                     # used as reference date for firstday
+	  -e, [--es-url=Elasticsearch endpoint for Logstash]  # or ES_URL environment variable
+
+All times will be relative to the timezone of the machine on which you are running lstash.
+
+## Elasticsearch configuration
+
+By default `lstash` will connnect to Elasticsearch on your localhost as `http://localhost:9200`. To connect
+to a different server you can set the `ES_URL` environment variable. URL scheme `http` and port `9200` are default
+and may be omitted.
+
+Example
+
+    export ES_URL=log.mydomain.com
+    lstash count program:haproxy
+
+Or
+
+	lstash count program:haproxy --es-url log.mydomain.com
+
+## Examples
+
+Count the number of haproxy log messages matching QUERY from Aug 1 at midnight (0:00 am) upto (not including) Aug 2 at midnight (0:00 am).
+
+    lstash count program:haproxy --from "Aug 1" --to "Aug 2"
+
+Grep all haproxy log messages using for one day (Aug 24 1 0:00 am upto and including Aug 2 23:59).
+
+    lstash grep program:haproxy --from "Aug 1" --to "Aug 2"
+
+Assuming today is Sep 1 2014. Count all haproxy log messages in the previous month.
+
+	lstash count program:haproxy --anchor yesterday --from firstday --to today -d
+	time range: [2014-08-01 00:00:00 +0200..2014-09-01 00:00:00 +0200]
+	logstash-2014.07.31: 1 
+	logstash-2014.08.01: 13 
+	logstash-2014.08.02: 14 
+	logstash-2014.08.03: 1654 
+	logstash-2014.08.04: 6 
+	logstash-2014.08.05: 20 
+	logstash-2014.08.06: 219 
+	logstash-2014.08.07: 32 
+	logstash-2014.08.08: 14 
+	logstash-2014.08.09: 28 
+	logstash-2014.08.10: 799 
+	logstash-2014.08.11: 18 
+	logstash-2014.08.12: 8 
+	logstash-2014.08.13: 23 
+	logstash-2014.08.14: 25 
+	logstash-2014.08.15: 69 
+	logstash-2014.08.16: 19 
+	logstash-2014.08.17: 1160 
+	logstash-2014.08.18: 284 
+	logstash-2014.08.19: 61 
+	logstash-2014.08.20: 26 
+	logstash-2014.08.21: 16 
+	logstash-2014.08.22: 145 
+	logstash-2014.08.23: 72 
+	logstash-2014.08.24: 792 
+	logstash-2014.08.25: 31 
+	logstash-2014.08.26: 33 
+	logstash-2014.08.27: 51 
+	logstash-2014.08.28: 8 
+	logstash-2014.08.29: 23 
+	logstash-2014.08.30: 25 
+	logstash-2014.08.31: 69 
+	5633
+
+## Using lstash as a gem in your project
+
+Add this line to your application's Gemfile:
+
+    gem 'lstash'
+
+And then execute:
+
+    $ bundle
+
+Usage:
+
+	$ bundle console
+
+	# connect to elasticsearch and create the Lstash client
+	elasticsearch = Elasticsearch::Client.new(url: 'log.mydomain.com')
+	client = Lstash::Client.new(elasticsearch)
+
+	# create the query
+	query = Lstash::Query.new('program:haproxy', from: 'today', to: 'now')
+
+	# count
+	client.count(query)
+
+	# grep
+	client.grep(query) do |message|
+	  puts message
+	end
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/lstash

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'lstash/cli'
+
+Lstash::CLI.start(ARGV)

data/lib/lstash/cli.rb

@@ -0,0 +1,77 @@
+# external dependencies
+require 'thor'
+require 'patron' # use Patron HTTP library for optimal performance
+require 'elasticsearch'
+
+# local files we need
+require 'lstash/query'
+require 'lstash/client'
+
+module Lstash
+
+  class CLI < Thor
+
+    class_option :from,   :banner => 'start of time range', :aliases => '-f', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
+    class_option :to,     :banner => 'end of time range',   :aliases => '-t', :desc => "date/time, 'now', 'today', 'yesterday', or 'firstday'"
+    class_option :anchor, :banner => 'anchor date/time',    :aliases => '-a', :desc => "used as reference date for firstday"
+    class_option :es_url, :banner => 'Elasticsearch endpoint for Logstash', :aliases => '-e', :desc => "or ES_URL environment variable"
+    class_option :debug,  :banner => 'debug log to stderr', :aliases => '-d', :type => :boolean
+
+    long_desc <<-LONGDESC
+      Grep log messages matching the QUERY from Logstash in ascending timestamp order
+      and output to stdout. QUERY can use Apache Lucene query parser syntax.
+
+      Example to grep HAProxy log messages from the beginning of this month upto now
+
+        lstash grep 'program:haproxy' --from firstday --to now
+    LONGDESC
+    desc "grep QUERY", "grep log messages from Logstash"
+    def grep(query_string)
+      run_command(query_string) do |es_client, query|
+        Lstash::Client.new(es_client, options).grep(query) do |message|
+          puts message
+        end
+      end
+    end
+
+    long_desc <<-LONGDESC
+      Count log messages matching the QUERY from Logstash and output this count to stdout.
+      QUERY can use Apache Lucene query parser syntax.
+
+      Example to count the number of HAProxy log messages in yesterdays month.
+
+        lstash count 'program:haproxy' --from firstday --to today --anchor yesterday
+    LONGDESC
+    desc "count QUERY", "count number of log messages matching the QUERY"
+    def count(query_string)
+      run_command(query_string) do |es_client, query|
+        count  = Lstash::Client.new(es_client, options).count(query)
+        puts count
+      end
+    end
+
+    private
+
+    def run_command(query_string)
+      es_client = ::Elasticsearch::Client.new(
+        url: options[:es_url] || ENV['ES_URL'] || 'localhost',
+        log: !!ENV['DEBUG']
+      )
+      query  = Lstash::Query.new(query_string, options)
+
+      yield es_client, query
+
+    rescue Exception => e
+      raise Thor::Error.new(e.message)
+    end
+
+    protected
+
+    # Make sure we exit on failure with an error code
+    def self.exit_on_failure?
+      true
+    end
+
+  end
+
+end

data/lib/lstash/client.rb

@@ -0,0 +1,94 @@
+require 'logger'
+require 'date'
+require 'hashie'
+
+class NullLogger < Logger
+  def initialize(*args); end
+  def add(*args, &block); end
+end
+
+module Lstash
+
+  class Client
+
+    class ConnectionError < StandardError; end
+
+    PER_PAGE = 5000 # best time, lowest resource usage
+
+    def initialize(es_client, options = {})
+      raise ConnectionError, "No elasticsearch client specified" if es_client.nil?
+
+      @es_client = es_client
+      @logger    = options[:logger] || (options[:debug] ? debug_logger : NullLogger.new)
+    end
+
+    def count(query)
+      @logger.debug "time range: [%s..%s]" % [query.time_range.from, query.time_range.to]
+
+      count = 0
+      query.indices.each do |index|
+        count += count_messages(index, query)
+      end
+
+      count
+    end
+
+    def grep(query)
+      @logger.debug "time range: [%s..%s]" % [query.time_range.from, query.time_range.to]
+
+      query.indices.each do |index|
+        grep_messages(index, query) do |message|
+          yield message if block_given?
+        end
+      end
+    end
+
+    private
+
+    def count_messages(index, query)
+      result = Hashie::Mash.new @es_client.send(:count,
+        index: index,
+        body:  query.body[:query]
+      )
+      @logger.debug "count #{index}: #{result['count']} "
+      result['count']
+    end
+
+    def grep_messages(index, query)
+      messages = nil
+      scroll_params = {}
+      offset = 0
+      method = :search
+      while (messages.nil? || messages.count > 0) do
+        result = Hashie::Mash.new @es_client.send(method, {
+          index: index,
+          scroll: '10m',
+          body: query.body.merge(from: offset, size: PER_PAGE),
+        }.merge(scroll_params))
+
+        messages = result.hits.hits
+
+        offset += messages.count
+        scroll_params = {scroll_id: result._scroll_id}
+
+        messages.each do |h|
+          yield h._source.message if block_given?
+        end
+
+        method = :scroll
+      end
+      @logger.debug "grep #{index}: #{offset}"
+      Hashie::Mash.new @es_client.clear_scroll(scroll_params)
+    end
+
+    def debug_logger
+      logger = Logger.new(STDERR)
+      logger.formatter = proc do |severity, datetime, progname, msg|
+        "#{msg}\n"
+      end
+      logger
+    end
+
+  end
+
+end

data/lib/lstash/query.rb

@@ -0,0 +1,159 @@
+require 'time'
+require 'date'
+require 'ostruct'
+
+module Lstash
+
+  class Query
+
+    class FormatError < StandardError; end
+    class QueryMissing < StandardError; end
+
+    LOGSTASH_PREFIX = 'logstash-'
+    WILDCARD_QUERY = '*'
+
+    def initialize(query = nil, arguments = {})
+      @query = query
+
+      @anchor = time_parse(arguments[:anchor], 'today')
+      @from   = time_parse(arguments[:from],   'today')
+      @to     = time_parse(arguments[:to],     'now')
+
+      @to = Time.now if @to > Time.now # prevent accessing non-existing times / indices
+    end
+
+    def time_range
+      OpenStruct.new(from: @from, to: @to)
+    end
+
+    def date_range
+      (@from.utc.to_date .. @to.utc.to_date)
+    end
+
+    def indices
+      date_range.map { |d| "#{LOGSTASH_PREFIX}#{d.strftime('%Y.%m.%d')}" }
+    end
+
+    def body
+      {
+        sort: sort_order,
+
+        # return in order of ascending timestamp
+        query: {
+          filtered: {
+            query:  es_query,
+            filter: es_filter
+          }
+        }
+      }
+    end
+
+    private
+
+    def time_parse(time_string, default)
+      time_string = time_string.strip rescue nil
+      time_string ||= default
+      case time_string
+      when 'firstday'
+        midnight_at_beginning_of_month
+      when 'now'
+        Time.now
+      when 'today'
+        midnight_today
+      when 'yesterday'
+        midnight_yesterday
+      else
+        Time.parse(time_string)
+      end
+    rescue ArgumentError
+      raise FormatError, "Invalid time format: #{time_string}"
+    end
+
+    def query
+      q = @query.dup.strip rescue ''
+      q = WILDCARD_QUERY if q.empty?
+      q
+    end
+
+    def sort_order
+      [ { '@timestamp' => { order: 'asc' } } ]
+    end
+
+    def es_query
+      {
+        bool: {
+          should: [
+            {
+              query_string: {
+                query: query
+              }
+            }
+          ]
+        }
+      }
+    end
+
+    def es_filter
+      {
+        bool: {
+          must: [
+            range: {
+              '@timestamp' => {
+                from: to_msec(time_range.from),
+                to:   to_msec(time_range.to)
+              }
+            },
+            # fquery: {
+            #   query: {
+            #     query_string: {
+            #       query: query
+            #     }
+            #   }
+            # }
+          ],
+          # must_not: [
+          #   fquery: {
+          #     query: {
+          #       query_string: {
+          #         query: query
+          #       }
+          #     }
+          #   }
+          # ],
+          # should: [
+          #   fquery: {
+          #     query: {
+          #       query_string: {
+          #         query: query
+          #       }
+          #     }
+          #   }
+          # ]
+        }
+      }
+    end
+
+    # Return the date of the first day of date's month
+    def midnight_at_beginning_of_month
+      Date.new(anchor_time.year, anchor_time.month, 1).to_time
+    end
+
+    def midnight_today
+      Date.today.to_time
+    end
+
+    def midnight_yesterday
+      (Date.today-1).to_time
+    end
+
+    def anchor_time
+      @anchor
+    end
+
+    def to_msec(time)
+      time.to_i * 1000
+    end
+
+  end
+
+end

data/lib/lstash/version.rb

@@ -0,0 +1,3 @@
+module Lstash
+  VERSION = "0.0.6"
+end

data/lib/lstash.rb

@@ -0,0 +1,8 @@
+require 'elasticsearch'
+require 'lstash/version'
+require 'lstash/query'
+require 'lstash/client'
+
+module Lstash
+  # Your code goes here...
+end

data/lstash.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'lstash/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lstash"
+  spec.version       = Lstash::VERSION
+  spec.authors       = ["Klaas Jan Wierenga"]
+  spec.email         = ["k.j.wierenga@gmail.com"]
+  spec.description   = %q{Count or grep log messages in a specified time range from a Logstash Elasticsearch server.}
+  spec.summary       = %q{The lstash gem allows you to count or grep log messages in a specific time range from a Logstash Elasticsearch server. }
+  spec.homepage      = "http://bitbucket.org/kjwierenga/lstash"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  spec.extra_rdoc_files = [ 'LICENSE.txt', 'README.md' ]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-its"
+  spec.add_development_dependency "rspec-autotest"
+  spec.add_development_dependency "autotest-standalone"
+  spec.add_development_dependency "autotest-fsevent"
+  spec.add_development_dependency "timecop"
+
+  spec.add_dependency "patron"
+  spec.add_dependency "elasticsearch", "~> 0.4"
+  spec.add_dependency "hashie"
+  spec.add_dependency "thor"
+end

data/spec/lstash/cli_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+require 'lstash/cli'
+
+class Lstash::CLI < Thor
+  def self.exit_on_failure?
+    false
+  end
+end
+
+describe Lstash::CLI do
+
+  context "options" do
+    subject { Lstash::CLI.options(args) }
+
+    let(:args) { %w(extract --time from --to to --one --two --three --four) }
+
+    its(:keys) { should eq args }
+  end
+
+  context "count" do
+
+    context "with valid arguments" do
+      let(:args) { %w(count "program:haproxy" --es-url localhost) }
+
+      it "should succeed" do
+        client = double('client')
+
+        allow(Lstash::Client).to receive(:new).and_return(client)
+        allow(client).to receive(:count).and_return(100)
+
+        output = capture_stdout { Lstash::CLI.start(args) }
+
+        expect(output).to eq "100\n"
+      end
+    end
+
+    context "with invalid --es-url" do
+      let(:args) { %w(count "program:haproxy" --es-url '') }
+
+      it "should print error message" do
+        output = capture_stderr { Lstash::CLI.start(args) }
+
+        expect(output).to eq "the scheme http does not accept registry part: '':9200 (or bad hostname?)\n"
+      end
+    end
+
+    context "without query" do
+      let(:args) { %w() }
+      it "should print help message" do
+        output = capture_stdout { Lstash::CLI.start(args) }
+
+        expect(output).to match("Commands:\n  rspec count QUERY")
+      end
+    end
+
+    context "with anchor date" do
+      let(:args) { %w(count program:haproxy --from firstday --to today --anchor yesterday) }
+
+      it "should succeed" do
+        Timecop.freeze('2014-08-01 14:58') do
+          es_client = double('es_client')
+
+          allow(Elasticsearch::Client).to receive(:new) { es_client }
+
+          expect(es_client).to receive(:count).with(satisfy { |args|
+            expect_time_range(args, [
+              Time.parse('2014-07-01').to_i*1000,
+              Time.parse('2014-08-01').to_i*1000
+            ])
+          })
+
+          Lstash::CLI.start(args)
+        end
+      end
+    end
+
+    context "without anchor date" do
+      let(:args) { %w(count program:haproxy --from yesterday --to today) }
+
+      it "should succeed" do
+        Timecop.freeze('2014-08-01 14:58') do
+          es_client = double('es_client')
+
+          allow(Elasticsearch::Client).to receive(:new) { es_client }
+
+          expect(es_client).to receive(:count).with(satisfy { |args|
+            expect_time_range(args, [
+              Time.parse('2014-07-31').to_i*1000,
+              Time.parse('2014-08-01').to_i*1000
+            ])
+          }).and_return([{count:1}])
+
+          Lstash::CLI.start(args)
+        end
+      end
+    end
+
+  end
+
+  private
+
+  def expect_time_range(args, time_range)
+    expect(args[:body][:filtered][:filter][:bool][:must].first[:range]['@timestamp'][:from]).to eq time_range.first
+    expect(args[:body][:filtered][:filter][:bool][:must].first[:range]['@timestamp'][:to]).to   eq time_range.last
+  end
+
+end

data/spec/lstash/client_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+require 'lstash/client'
+
+describe Lstash::Client do
+
+  let(:es_client) { double('es_client') }
+  subject { Lstash::Client.new(es_client) }
+
+  it "should initialize properly" do
+    expect(subject).not_to be nil
+  end
+
+  context "with query" do
+
+    let(:query) { double('query', time_range: OpenStruct.new) }
+
+    context "count" do
+      it "should return number of messages matching query" do
+        allow(query).to receive(:indices).and_return (['logstash-2014-08-01', 'logstash-2014-08-02'])
+        allow(query).to receive(:body).and_return ({})
+
+        allow(es_client).to receive(:count).and_return({'count' => 100},{'count' => 100})
+
+        expect(subject.count(query)).to eq 200
+      end
+    end
+
+    context "grep" do
+      let(:query) { double('query', time_range: OpenStruct.new) }
+      
+      it "should return the messages matching the query" do
+        allow(query).to receive(:indices).and_return (['logstash-2014-08-01', 'logstash-2014-08-02'])
+        allow(query).to receive(:body).and_return ({})
+        
+        allow(es_client).to receive(:search).and_return(
+          hits([
+            'this is the first log line',
+            'this is the second log line'
+          ])
+        )
+
+        allow(es_client).to receive(:scroll).and_return(hits([]))
+
+        allow(es_client).to receive(:clear_scroll)
+
+        subject.grep(query)
+      end
+    end
+
+  end
+
+  def hits(messages)
+    {
+      hits: {
+        hits: messages.map { |m| { _source: { message: m }}}
+      }
+    }
+  end
+
+end

data/spec/lstash/query_spec.rb

@@ -0,0 +1,145 @@
+require 'spec_helper'
+require 'lstash/query'
+
+describe Lstash::Query do
+
+  context "running on 2014-08-03" do
+    let(:time)  { '2014-08-03 15:54:33' }
+    let(:query) { nil }
+    let(:options) { {} }
+
+    subject { Lstash::Query.new(query, options) }
+
+    before { Timecop.freeze(Time.parse(time)) }
+    after  { Timecop.return }
+
+    it { should_not be nil }
+    # it "should initialize properly" do
+    #   expect(subject).not_to be nil
+    # end
+
+    its('time_range.from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+    its('time_range.to')   { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+
+    its(:date_range) { should eq (Date.parse('2014-08-02')..Date.parse('2014-08-03')) }
+
+    its(:indices) { should eq [ "logstash-2014.08.02", "logstash-2014.08.03" ] }
+
+    context "with specific query" do
+      let(:query) { 'program:haproxy' }
+      its(:query) { should eq query }
+    end
+
+    context "without query" do
+      let(:query) { nil }
+      its(:query) { should eql '*' }
+    end
+
+    context "from 'yesterday'" do
+      let(:options) { { from: 'yesterday' }}
+      its('time_range.from') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
+    end
+
+    context "from 'today'" do
+      let(:options) { { from: 'today' }}
+      its('time_range.from') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+    end
+
+    context "from 'now'" do
+      let(:options) { { from: 'now' }}
+      its('time_range.from') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+    end
+
+    context "to 'yesterday'" do
+      let(:options) { { to: 'yesterday' }}
+      its('time_range.to') { should eq Time.parse('2014-08-02 00:00:00.000000000 +0200') }
+    end
+
+    context "to 'today'" do
+      let(:options) { { to: 'today' }}
+      its('time_range.to') { should eq Time.parse('2014-08-03 00:00:00.000000000 +0200') }
+    end
+
+    context "to 'now'" do
+      let(:options) { { to: 'now' }}
+      its('time_range.to') { should eq Time.parse('2014-08-03 15:54:33.000000000 +0200') }
+    end
+
+    context "from 'firstday'" do
+
+      let(:options) { { from: 'firstday' } }
+      its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+
+      context "anchor 'yesterday'" do
+        let(:anchor) { 'yesterday' }
+        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+      end
+
+      context "anchor 'today'" do
+        let(:anchor) { 'today' }
+        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+      end
+
+      context "anchor '2014-07-17'" do
+        let(:options) { { from: 'firstday', anchor: '2014-07-17' } }
+        its('time_range.from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
+      end
+
+      context "date range" do
+        let(:options) { { from: 'firstday', anchor: 'yesterday' } }
+        its(:date_range) { should eq (Date.parse('2014-07-31')..Date.parse('2014-08-03')) }
+      end
+
+      context "indices" do
+        let(:options) { { from: 'firstday', anchor: 'yesterday' } }
+        its(:indices) {
+          should eq [
+            "logstash-2014.07.31",
+            "logstash-2014.08.01",
+            "logstash-2014.08.02",
+            "logstash-2014.08.03",
+          ]
+        }
+      end
+
+    end
+
+    context "body" do
+      its(:body) { should eq ({
+        :sort => [{"@timestamp"=>{:order=>"asc"}}],
+        :query => {:filtered=>{
+          :query => { :bool => { :should => [ { :query_string => { :query=>"*" }}]}},
+          :filter=> { :bool => { :must   => [ { :range => { "@timestamp" => { :from => 1407016800000, :to => 1407074073000}}}]}}}}
+      })}
+    end
+
+  end
+
+  context "running on 2014-08-01" do
+    let(:time)  { '2014-08-01 12:53:03' }
+    let(:query) { nil }
+    let(:options) { {} }
+
+    subject { Lstash::Query.new(query, options) }
+
+    before { Timecop.freeze(Time.parse(time)) }
+    after  { Timecop.return }
+
+    context "from 'firstday' with 'yesterday' anchor" do
+      let(:options) { { anchor: 'yesterday', from: 'firstday' } }
+
+        its('time_range.from') { should eq Time.parse('2014-07-01 00:00:00.000000000 +0200') }
+        its('time_range.to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
+    end
+
+    context "from 'firstday' with default 'today' anchor" do
+      let(:options) { { from: 'firstday', to: 'now' } }
+
+        its('time_range.from') { should eq Time.parse('2014-08-01 00:00:00.000000000 +0200') }
+        its('time_range.to')   { should eq Time.parse('2014-08-01 12:53:03.000000000 +0200') }
+    end
+
+  end
+
+end
+

data/spec/lstash_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'lstash/cli'
+
+describe Lstash do
+  it 'should have a version number' do
+    expect(Lstash::VERSION).not_to be nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,33 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'lstash'
+
+require 'rspec/its'
+
+require 'timecop'
+
+ENV['ES_URL'] = nil
+ENV['TZ'] = 'Europe/Amsterdam' # Test in a specific timezone.
+
+RSpec.configure do |config|
+  config.order = 'random'
+end
+
+require 'stringio'
+
+def capture_stdout(&blk)
+  old = $stdout
+  $stdout = fake = StringIO.new
+  blk.call
+  fake.string
+ensure
+  $stdout = old
+end
+
+def capture_stderr(&blk)
+  old = $stderr
+  $stderr = fake = StringIO.new
+  blk.call
+  fake.string
+ensure
+  $stderr = old
+end

metadata

@@ -0,0 +1,245 @@
+--- !ruby/object:Gem::Specification
+name: lstash
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+platform: ruby
+authors:
+- Klaas Jan Wierenga
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-autotest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: autotest-standalone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: autotest-fsevent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: patron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Count or grep log messages in a specified time range from a Logstash
+  Elasticsearch server.
+email:
+- k.j.wierenga@gmail.com
+executables:
+- lstash
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- .autotest
+- .gitignore
+- .rspec
+- .ruby-gemset
+- .ruby-version
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/lstash
+- lib/lstash.rb
+- lib/lstash/cli.rb
+- lib/lstash/client.rb
+- lib/lstash/query.rb
+- lib/lstash/version.rb
+- lstash.gemspec
+- spec/lstash/cli_spec.rb
+- spec/lstash/client_spec.rb
+- spec/lstash/query_spec.rb
+- spec/lstash_spec.rb
+- spec/spec_helper.rb
+homepage: http://bitbucket.org/kjwierenga/lstash
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: The lstash gem allows you to count or grep log messages in a specific time
+  range from a Logstash Elasticsearch server.
+test_files:
+- spec/lstash/cli_spec.rb
+- spec/lstash/client_spec.rb
+- spec/lstash/query_spec.rb
+- spec/lstash_spec.rb
+- spec/spec_helper.rb