peruse 0.4.0

data/spec/chained_search_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+require 'shared/time_stubs'
+require 'shared/peruse_stubs'
+
+describe 'chained searches' do
+  include_context "time stubs"
+  include_context "peruse stubs"
+
+  it 'should parse last 24h & foo_type=bar & baz="fez" & host=27.224.123.110' do
+    result = Peruse.search 'last 24h & foo_type=bar & baz="fez" & host=27.224.123.110'
+    expected = Peruse::Helper.filter_builder({
+      and: [
+        Peruse::Helper.range_builder(
+          (@time - 24.hours).utc.to_datetime.iso8601(3),
+          @time.utc.to_datetime.iso8601(3)
+        ),
+        Peruse::Helper.query_builder('foo_type:bar'),
+        Peruse::Helper.query_builder('baz:"fez"'),
+        Peruse::Helper.query_builder('host:27.224.123.110')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse last 24h & (foo_type=bar AND baz="fez" AND host=27.224.123.110)' do
+    result = Peruse.search 'last 24h & (foo_type=bar AND baz="fez" AND host=27.224.123.110)'
+    expected = Peruse::Helper.filter_builder({
+      and: [
+        Peruse::Helper.range_builder(
+          (@time - 24.hours).utc.to_datetime.iso8601(3),
+          @time.utc.to_datetime.iso8601(3)
+        ),
+        Peruse::Helper.query_builder('foo_type:bar'),
+        Peruse::Helper.query_builder('baz:"fez"'),
+        Peruse::Helper.query_builder('host:27.224.123.110')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+end

data/spec/field_value_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+describe 'field / value searches' do
+
+  it 'should parse _foo.@bar=baz' do
+    result = Peruse.search '_foo.@bar=baz'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.query_builder('_foo.@bar:baz')
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse _foo.@bar=(baz)' do
+    result = Peruse.search '_foo.@bar=(baz)'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.query_builder('_foo.@bar:(baz)')
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse _foo.@fields.@bar="bar baz"' do
+    result = Peruse.search '_foo.@fields.@bar="bar baz"'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.query_builder('_foo.@fields.@bar:"bar baz"')
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse _foo-barcr@5Y_+f!3*(name=bar' do
+    result = Peruse.search '_foo-barcr@5Y_+f!3*(name=bar'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.query_builder('_foo-barcr@5Y_+f!3*(name:bar')
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse foo=bar-baz' do
+    result = Peruse.search 'foo=bar-baz'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.query_builder('foo:bar-baz')
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse !src_ip=0.0.0.0' do
+    result = Peruse.search '!src_ip=0.0.0.0'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.query_builder('!src_ip:0.0.0.0')
+    )
+    expect(result).to eq(expected)
+  end
+end

data/spec/indices_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe 'the indices command' do
+  it 'should parse indices foo,bar,baz' do
+    result = Peruse.search 'indices foo,bar,baz'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.indices_builder(%w(foo bar baz))
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse indices foo , bar , baz' do
+    result = Peruse.search 'indices foo , bar , baz'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.indices_builder(%w(foo bar baz))
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse indices foo , bar , baz & key = value' do
+    result = Peruse.search 'indices foo , bar , baz & key = value'
+    expected = Peruse::Helper.filter_builder(
+      and: [
+        Peruse::Helper.indices_builder(%w(foo bar baz)),
+        Peruse::Helper.query_builder('key:value')
+      ]
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse key = value & indices foo , bar , baz' do
+    result = Peruse.search 'key = value & indices foo , bar , baz'
+    expected = Peruse::Helper.filter_builder(
+      and: [
+        Peruse::Helper.query_builder('key:value'),
+        Peruse::Helper.indices_builder(%w(foo bar baz))
+      ]
+    )
+    expect(result).to eq(expected)
+  end
+end

data/spec/last_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+require 'shared/time_stubs'
+require 'shared/peruse_stubs'
+
+describe 'the last command' do
+  include_context "time stubs"
+  include_context "peruse stubs"
+
+  it 'should parse last 24h' do
+    result = Peruse.search 'last 24h'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        (@time - 24.hours).utc.to_datetime.iso8601(3),
+        @time.utc.to_datetime.iso8601(3)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse last 24d' do
+    result = Peruse.search 'last 24d'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        (@time - 24.days).utc.to_datetime.iso8601(3),
+        @time.utc.to_datetime.iso8601(3)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse last 24w' do
+    result = Peruse.search 'last 24w'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        (@time - 24.weeks).utc.to_datetime.iso8601(3),
+        @time.utc.to_datetime.iso8601(3)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse last 24s' do
+    result = Peruse.search 'last 24s'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        (@time - 24.seconds).utc.to_datetime.iso8601(3),
+        @time.utc.to_datetime.iso8601(3)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse last 24m' do
+    result = Peruse.search 'last 24m'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        (@time - 24.minutes).utc.to_datetime.iso8601(3),
+        @time.utc.to_datetime.iso8601(3)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse last 1h & @fields.foo.@field=bar' do
+    result = Peruse.search 'last 1h & @fields.foo.@field=bar'
+    expected = Peruse::Helper.filter_builder(
+      and: [
+        Peruse::Helper.range_builder(
+          (@time - 1.hour).utc.to_datetime.iso8601(3),
+          @time.utc.to_datetime.iso8601(3)
+        ),
+        Peruse::Helper.query_builder('@fields.foo.@field:bar')
+      ]
+    )
+    expect(result).to eq(expected)
+  end
+end

data/spec/limit_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe 'the limit command' do
+  it 'should parse limit 1' do
+    result = Peruse.search 'limit 1'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.limit_builder(1)
+    )
+    expect(result).to eq(expected)
+  end
+end

data/spec/nested_search_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+require 'shared/time_stubs'
+require 'shared/peruse_stubs'
+
+describe 'nested searches' do
+  include_context "time stubs"
+  include_context "peruse stubs"
+  
+  before :each do
+    fake_results = {
+      foo: 'bar',
+      baz: 5,
+      arr: [ 0, 1, 2, 3 ],
+      :timestamp => @time.utc.to_datetime.iso8601(3)
+    }.to_json
+    allow_any_instance_of(Peruse::ResultSet).to receive(:eval).and_return(fake_results)
+  end
+
+  skip 'should transform' do
+    results = @transformer.apply @parser.parse('foo=`bar=baz|baz,fass,fdsd`')
+    expect(results.query).to eq({query:{filtered:{query:{query_string:{
+      query: 'foo:(5)'
+    }}}}})
+  end
+
+  skip 'should parse a nested basic search' do
+    @parsed = @parser.parse 'tshark.len = ` 226 | tshark.frame.time_epoch,tshark.ip.src`'
+    expect(@parsed[:field].to_s).to eq 'tshark.len'
+    expect(@parsed[:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:match].to_s).to eq '226'
+    expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+  end
+
+  skip 'should parse a nested regexp' do
+    @parsed = @parser.parse 'tshark.len = ` cif.malicious_ips=/foo/ | tshark.frame.time_epoch,tshark.ip.src`'
+    expect(@parsed[:field].to_s).to eq 'tshark.len'
+    expect(@parsed[:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:field].to_s).to eq 'cif.malicious_ips'
+    expect(@parsed[:value][:initial_query][:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:value].to_s).to eq '/foo/'
+    expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+  end
+
+  skip 'should parse a nested basic boolean' do
+    @parsed = @parser.parse 'tshark.len = `(foo OR bar) | tshark.frame.time_epoch,tshark.ip.src`'
+    expect(@parsed[:field].to_s).to eq 'tshark.len'
+    expect(@parsed[:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:match].to_s).to eq '(foo OR bar) '
+    expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+  end
+
+  skip 'should parse a nested field / value boolean' do
+    @parsed = @parser.parse 'tshark.len = `baz=(foo OR bar AND (bar OR fez)) | tshark.frame.time_epoch,tshark.ip.src`'
+    expect(@parsed[:field].to_s).to eq 'tshark.len'
+    expect(@parsed[:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:field].to_s).to eq 'baz'
+    expect(@parsed[:value][:initial_query][:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:value].to_s).to eq '(foo OR bar AND (bar OR fez)) '
+    expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+  end
+
+  skip 'should parse a nested last standalone timerange' do
+    @parsed = @parser.parse 'tshark.len = `last 24h | tshark.frame.time_epoch,tshark.ip.src`'
+    expect(@parsed[:field].to_s).to eq 'tshark.len'
+    expect(@parsed[:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:timerange][:quantity].to_s).to eq '24'
+    expect(@parsed[:value][:initial_query][:timerange][:quantifier].to_s).to eq 'h'
+    expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+  end
+
+  skip 'should parse a nested last timerange and field / value pair' do
+    @parsed = @parser.parse 'tshark.len = `last 24h foo=bar | tshark.frame.time_epoch,tshark.ip.src`'
+    expect(@parsed[:field].to_s).to eq 'tshark.len'
+    expect(@parsed[:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:timerange][:quantity].to_s).to eq '24'
+    expect(@parsed[:value][:initial_query][:timerange][:quantifier].to_s).to eq 'h'
+    expect(@parsed[:value][:initial_query][:search][:field].to_s).to eq 'foo'
+    expect(@parsed[:value][:initial_query][:search][:op].to_s).to eq '='
+    expect(@parsed[:value][:initial_query][:search][:value].to_s).to eq 'bar'
+    expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+  end
+end

data/spec/regexp_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe 'regexp searches' do
+
+  it 'should parse foo=/blah foo/' do
+    result = Peruse.search 'foo=/blah foo/'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.regexp_builder(
+        'foo',
+        'blah foo'
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse foo=/blah\/ foo/' do
+    result = Peruse.search 'foo=/blah\/ foo/'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.regexp_builder(
+        'foo',
+        'blah\/ foo'
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse foo=/blah\. foo/' do
+    result = Peruse.search 'foo=/blah\. foo/'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.regexp_builder(
+        'foo',
+        'blah\. foo'
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse http.headers=/.*User\-Agent\: Microsoft\-WebDAV.*/' do
+    result = Peruse.search 'http.headers=/.*User\-Agent\: Microsoft\-WebDAV.*/'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.regexp_builder(
+        'http.headers',
+        '.*User\-Agent\: Microsoft\-WebDAV.*'
+      )
+    )
+    expect(result).to eq(expected)
+  end
+end

data/spec/shared/dummy_client.rb

@@ -0,0 +1,14 @@
+# Dummy Elasticsearch Client for specs
+
+module Elasticsearch
+  module Transport
+    class Client
+      def initialize(opts)
+      end
+
+      def search(payload)
+        payload[:body]
+      end
+    end
+  end
+end

data/spec/shared/peruse_stubs.rb

@@ -0,0 +1,5 @@
+shared_context "peruse stubs" do
+  before :each do
+    allow(Peruse).to receive(:timestamp_field).and_return(:@timestamp)
+  end
+end

data/spec/shared/time_stubs.rb

@@ -0,0 +1,12 @@
+require 'timecop'
+
+shared_context "time stubs" do
+  before do
+    Timecop.freeze(Time.local(2012))
+    @time = Time.now
+  end
+
+  after do
+    Timecop.return
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+require 'rspec'
+require 'peruse'
+require 'parslet/rig/rspec'
+require 'shared/dummy_client'
+
+# Print ascii_tree when exception occurs
+module Peruse
+  class ParserWrapper < Parser
+    def parse(query)
+      begin
+        super(query)
+      rescue Parslet::ParseFailed => failure
+        puts failure.cause.ascii_tree
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.before :all do
+    
+    # configure test instance of Peruse to use wrapper parser
+    Peruse.configure do |c|
+      c.parser = Peruse::ParserWrapper.new
+      c.transformer = Peruse::Transformer.new
+    end
+  end
+end

data/spec/window_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'shared/time_stubs'
+require 'shared/peruse_stubs'
+require 'chronic'
+
+describe 'the window command' do
+  include_context 'time stubs'
+  include_context 'peruse stubs'
+
+  it 'should parse window 3/11/13 to 3/12/13' do
+    result = Peruse.search 'window 3/11/13 to 3/12/13'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        Peruse::Helper.timestamp_format(Chronic.parse('3/11/13')),
+        Peruse::Helper.timestamp_format(Chronic.parse('3/12/13'))
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse window "last monday" to "last thursday"' do
+    result = Peruse.search 'window "last monday" to "last thursday"'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        Peruse::Helper.timestamp_format(Chronic.parse('last monday')),
+        Peruse::Helper.timestamp_format(Chronic.parse('last thursday'))
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse window -60m to -30m' do
+    result = Peruse.search 'window -60m to -30m'
+    expected = Peruse::Helper.filter_builder(
+      Peruse::Helper.range_builder(
+        Peruse::Helper.timestamp_format(@time - 60.minutes),
+        Peruse::Helper.timestamp_format(@time - 30.minutes)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse NOT window -60m to -30m' do
+    result = Peruse.search 'NOT window -60m to -30m'
+    expected = Peruse::Helper.filter_builder(
+      not: 
+        Peruse::Helper.range_builder(
+          Peruse::Helper.timestamp_format(@time - 60.minutes),
+          Peruse::Helper.timestamp_format(@time - 30.minutes)
+        )
+    )
+    expect(result).to eq(expected)
+  end
+end