plunk 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 355fabb9e6a73101c70a9990992abe5974626a76
+  data.tar.gz: d61e5388b448da8412899038e8b0e3a72f3a81ac
+SHA512:
+  metadata.gz: 94c4b45f0018ac30b45b9922625d7ca25f7c28e8ff7506f831ad70e246179812e55b66b794611dc2f86a22e41ba0f6f7f53087f0bc8e0ddfc7d12ce81f108593
+  data.tar.gz: e957db51d157857e99a2b69138d678d0375ad27e98c594c30fff19e355a628d5a16d36f09d5b88b646c3c9af8d4cba687c04ab7f652ff87e827a6743f70d6ade

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    plunk (0.0.0)
+      json
+      parslet
+      rest-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    blankslate (2.1.2.4)
+    diff-lcs (1.2.5)
+    json (1.8.1)
+    mime-types (2.0)
+    parslet (1.5.0)
+      blankslate (~> 2.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  plunk!
+  rspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Elbii
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+plunk
+=====
+
+Human-friendly query language for Elasticsearch

data/Rakefile

@@ -0,0 +1,5 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/plunk.rb

@@ -0,0 +1,3 @@
+class Plunk
+
+end

data/lib/plunk/elasticsearch.rb

@@ -0,0 +1,121 @@
+require 'json'
+require 'rest-client'
+
+# Wrapper for Elasticsearch API
+class Plunk::Elasticsearch
+  attr_accessor :host, :port, :fields
+
+  def initialize(opts={})
+    @scheme = opts[:scheme] || "http"
+    @host = opts[:host] || "localhost"
+    @port = opts[:port] || "9200"
+    @size = opts[:size] || "10000"
+    @endpoint = "#{@scheme}://#{@host}:#{@port}"
+  end
+
+  # Get list of all field mappings stored in ES
+  # TODO: cache response from ES
+  def available_fields
+    uri = URI.escape "#{@endpoint}/_mapping"
+    result = JSON.parse(RestClient.get(uri))
+
+    @fields = {}
+    Plunk::Elasticsearch.nested_values(result, 'properties')
+
+    @fields
+  end
+
+  def active_record_errors_for(query)
+    return " can't be blank." if query.blank?
+
+    uri = URI.escape "#{@endpoint}/_validate/query?explain=true"
+    response = RestClient.post(uri, build_ES_validator(query))
+
+    json = JSON.parse(response)
+
+    json['valid'] ? nil : json['explanations'].collect { |exp| exp['error'] }
+  end
+
+  #
+  # UTILITY METHODS
+  #
+  def self.search(query)
+    uri = URI.escape "#{@endpoint}/_search?size=#{@size}"
+
+    RestClient.post uri, build_ES_query(query)
+  end
+
+  # returns all values for all occurences of the given nested key
+  def self.nested_values(hash, key)
+    hash.each do |k, v|
+      if k == key
+        @fields.merge! v
+      else
+        nested_values(v, key) if v.is_a? Hash
+      end
+    end 
+  end
+
+  # nested field matcher
+  def self.extract_values(hash, keys)
+    @vals ||= []
+
+    hash.each_pair do |k, v|
+      if v.is_a? Hash
+        extract_values(v, keys)
+      elsif v.is_a? Array
+        v.flatten!
+        if v.first.is_a? Hash
+          v.each { |el| extract_values(el, keys) }
+        elsif keys.include? k
+          @vals += v
+        end
+      elsif keys.include? k
+        @vals << v
+      end
+    end
+
+    return @vals
+  end
+
+  def self.build_ES_validator(query)
+    if valid_json? query
+      # Strip the top-level "query" paramter since ES doesn't expect it
+      JSON.parse(query)['query'].to_json
+    else
+      <<-END
+        {
+          "query_string": {
+            "query": "#{query}"
+          }
+        }
+      END
+    end
+  end
+
+  def self.build_ES_query(query)
+    if valid_json? query
+      query
+    else
+      <<-END
+        {
+          "query": {
+            "query_string": {
+              "query": "#{query}"
+            }
+          }
+        }
+      END
+    end
+  end
+
+  def self.valid_json?(json)
+    begin
+      JSON.parse json
+      true
+    rescue JSON::ParserError
+      false
+    end
+  end
+end
+

data/lib/plunk/parser.rb

@@ -0,0 +1,84 @@
+require 'parslet'
+
+class Plunk::Parser < Parslet::Parser
+  # Single character rules
+  rule(:lparen)     { str('(') >> space? }
+  rule(:rparen)     { str(')') >> space? }
+  rule(:comma)      { str(',') >> space? }
+  rule(:digit)      { match('[0-9]') }
+  rule(:space)      { match('\s').repeat(1) }
+  rule(:space?)     { space.maybe }
+
+  # Numbers
+  rule(:integer)    { str('-').maybe >> digit.repeat(1) >> space? }
+  rule(:float)      {
+    str('-').maybe >> digit.repeat(1) >> str('.') >> digit.repeat(1) >> space?
+  }
+  rule(:number)     { integer | float }
+
+  # Field / value
+  rule(:identifier) { match['_@a-zA-Z.'].repeat(1) }
+  rule(:wildcard)   { match('[a-zA-Z0-9.*]').repeat(1) }
+  rule(:searchop)   { match('[=]').as(:op) >> space? }
+
+  # boolean operators search
+  rule(:concatop)   { (str('OR') | str('AND')) >> space? }
+  rule(:operator)   { match('[|]').as(:op) >> space? }
+  rule(:timerange)  {
+    integer.as(:quantity) >> match('s|m|h|d|w').as(:quantifier)
+  }
+
+  # Grammar parts
+  rule(:rhs) {
+      (regex | wildcard | integer | subsearch |
+        (lparen >> (space? >> (wildcard | integer) >>
+          (space >> concatop).maybe).repeat(1) >> rparen))
+  }
+
+  rule(:regex) {
+    str('/') >> match('[^/]').repeat >> str('/')
+  }
+
+  rule(:search) {
+    identifier.as(:field) >> space? >> searchop >> space? >>
+      rhs.as(:value) | rhs.as(:match)
+  }
+
+  rule(:last) {
+    str("last") >> space >> timerange.as(:timerange) >> space >>
+    search.as(:search)
+  }
+
+  rule(:binaryop) {
+    (paren | search).as(:left) >> space? >> operator >> job.as(:right)
+  }
+
+  rule(:subsearch) {
+    str('`') >> space? >> nested_search >> str('`')
+  }
+
+  rule(:nested_search) {
+    match('[^|]').repeat.as(:term) >> str('|') >> space? >>
+    match('[^`]').repeat.as(:extractors)
+  }
+
+  rule(:joined_search) {
+    
+  }
+
+  rule(:paren) {
+    lparen >> space? >> job >> space? >> rparen
+  }
+
+  rule(:job) {
+    binaryop | paren | last | search
+  }
+
+  # root :job
+  rule(:plunk_query) {
+    job >> (space >> job).repeat
+  }
+
+  root :plunk_query
+end
+

data/lib/plunk/result_set.rb

@@ -0,0 +1,38 @@
+require_relative 'elasticsearch'
+
+class Plunk::ResultSet
+  attr_accessor :query
+
+  def initialize(opts=nil)
+    if opts
+      @query = { query: { }}
+      
+      if opts[:query_string]
+        @query[:query][:query_string] = { query: opts[:query_string] }
+      end
+
+      if opts[:start_time] and opts[:end_time]
+        @query[:query][:range] = {
+          '@timestamp' => {
+            gte: opts[:start_time],
+            lte: opts[:end_time]
+          }
+        }
+      end
+    end
+  end
+
+  def *(rs)
+    self.join(rs)
+  end
+
+  def join(rs)
+    ResultSet.new("[ #{@query} * #{rs.query} ]")
+  end
+
+  def eval
+    return unless @query
+    Elasticsearch.search(@query.to_json)
+  end
+end
+

data/lib/plunk/transformer.rb

@@ -0,0 +1,64 @@
+require 'parslet'
+
+class Plunk::Transformer < Parslet::Transform
+
+  rule(match: simple(:value)) do
+    ResultSet.new(query_string: "#{value}")
+  end
+
+  rule(
+    field: simple(:field),
+    value: {
+      term: simple(:term),
+      extractors: simple(:extractors)
+    },
+    op: '=') do
+
+    rs = ResultSet.new(query_string: "#{field}:#{term}")
+
+    json = JSON.parse rs.eval
+    values = Elasticsearch.extract_values json, extractors.to_s.split(',') 
+
+    if values.empty?
+      ResultSet.new
+    else
+      ResultSet.new(query_string: "(#{values.uniq.join(' OR ')})")
+    end
+  end
+
+  rule(field: simple(:field), value: simple(:value), op: '=') do
+    ResultSet.new(query_string: "#{field}:#{value}")
+  end
+
+  rule(
+    search: simple(:query),
+    timerange: {
+      quantity: simple(:quantity),
+      quantifier: simple(:quantifier)
+    }) do
+
+    int_quantity = quantity.to_s.to_i
+
+    start_time =
+      case quantifier
+      when 's'
+        int_quantity.seconds.ago
+      when 'm'
+        int_quantity.minutes.ago
+      when 'h'
+        int_quantity.hours.ago
+      when 'd'
+        int_quantity.days.ago
+      when 'w'
+        int_quantity.weeks.ago
+      end
+
+    end_time = Time.now.utc.to_datetime
+
+    ResultSet.new(
+      query_string: query,
+      start_time: start_time,
+      end_time: end_time)
+  end
+end
+

data/plunk.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name                    = "plunk"
+  s.version                 = "0.0.0"
+  s.date                    = "2013-11-26"
+  s.add_runtime_dependency  "json"
+  s.add_runtime_dependency  "parslet"
+  s.add_runtime_dependency  "rest-client"
+  s.add_development_dependency "rspec"
+  s.summary                 = "Elasticsearch query language"
+  s.description             = "Human-friendly query language for Elasticsearch"
+  s.authors                 = ["Ram Mehta", "Jamil Bou Kheir"]
+  s.email                   = ["jamil@elbii.com", "ram.mehta@gmail.com"]
+  s.files                   = `git ls-files`.split("\n")
+  s.homepage                = "https://github.com/elbii/plunk"
+  s.license                 = "MIT"
+end

data/spec/elasticsearch_spec.rb

@@ -0,0 +1,15 @@
+require 'plunk'
+require 'plunk/elasticsearch'
+
+describe Plunk::Elasticsearch do
+  before :all do
+    @elasticsearch = Plunk::Elasticsearch.new
+  end
+
+  context 'test field mapping' do
+    it 'should successfully list all fields' do
+      fields = @elasticsearch.available_fields
+      expect(fields).to be_a Hash
+    end
+  end
+end

data/spec/parser_spec.rb

@@ -0,0 +1,101 @@
+require 'plunk'
+require 'plunk/parser'
+
+# Print ascii_tree when exception occurs
+class Plunk::ParserWrapper < Plunk::Parser
+  def parse(query)
+    begin
+      super(query)
+    rescue Parslet::ParseFailed => failure
+      puts failure.cause.ascii_tree
+    end
+  end
+end
+
+describe Plunk::Parser do
+  before :all do
+    @parser = Plunk::ParserWrapper.new
+  end
+
+  context 'single-query searches' do
+    it 'should parse a single keyword' do
+      @parsed = @parser.parse 'bar'
+      expect(@parsed[:match].to_s).to eq 'bar'
+    end
+
+    it 'should parse a single field/value combo' do
+      @parsed = @parser.parse 'tshark.http.@src_ip=bar'
+      expect(@parsed[:field].to_s).to eq 'tshark.http.@src_ip'
+      expect(@parsed[:value].to_s).to eq 'bar'
+      expect(@parsed[:op].to_s).to eq '='
+    end
+
+    it 'should parse a single boolean expression' do
+      @parsed = @parser.parse '(bar OR car)'
+      expect(@parsed[:match].to_s).to eq '(bar OR car)'
+    end
+
+    it 'should parse a single field / value complex boolean expression' do
+      @parsed = @parser.parse 'ids.attackers=(bar OR car)'
+      expect(@parsed[:field].to_s).to eq 'ids.attackers'
+      expect(@parsed[:value].to_s).to eq '(bar OR car)'
+      expect(@parsed[:op].to_s).to eq '='
+    end
+
+    it 'should parse a single field / parenthesized value' do
+      @parsed = @parser.parse 'ids.attacker=(10.150.44.195)'
+      expect(@parsed[:field].to_s).to eq 'ids.attacker'
+      expect(@parsed[:value].to_s).to eq '(10.150.44.195)'
+      expect(@parsed[:op].to_s).to eq '='
+    end
+
+    it 'should parse a basic regex search' do
+      @parsed = @parser.parse 'foo=/blah foo/'
+      expect(@parsed[:field].to_s).to eq 'foo'
+      expect(@parsed[:value].to_s).to eq '/blah foo/'
+    end
+
+    context 'the last command' do
+      it 'should parse last command with a search' do
+        @parsed = @parser.parse 'last 24w tshark.@src_ip = bar'
+        expect(@parsed[:timerange][:quantity].to_s).to eq '24'
+        expect(@parsed[:timerange][:quantifier].to_s).to eq 'w'
+        expect(@parsed[:search][:field].to_s).to eq 'tshark.@src_ip'
+        expect(@parsed[:search][:value].to_s).to eq 'bar'
+      end
+    end
+
+    context 'chained searches' do
+      it 'should parse last command with a regex' do
+        @parsed = @parser.parse 'last 24w foo=/blah/'
+        expect(@parsed[:timerange][:quantity].to_s).to eq '24'
+        expect(@parsed[:timerange][:quantifier].to_s).to eq 'w'
+        expect(@parsed[:search][:field].to_s).to eq 'foo'
+        expect(@parsed[:search][:value].to_s).to eq '/blah/'
+      end
+    end
+
+    it 'should parse last command with boolean' do
+      @parsed = @parser.parse 'last 1h (foo OR bar)'
+      expect(@parsed[:search][:match].to_s).to eq '(foo OR bar)'
+    end
+
+    it 'should parse key/value with regex' do
+      @parsed = @parser.parse 'foo=bar fe.ip=/whodunnit/'
+      expect(@parsed[0][:field].to_s).to eq 'foo'
+      expect(@parsed[0][:value].to_s).to eq 'bar'
+      expect(@parsed[1][:field].to_s).to eq 'fe.ip'
+      expect(@parsed[1][:value].to_s).to eq '/whodunnit/'
+    end
+  end
+
+  context 'nested search' do
+    it 'should parse the nested search' do
+      @parsed = @parser.parse 'tshark.len = ` 226 | tshark.frame.time_epoch,tshark.ip.src`'
+      expect(@parsed[:field].to_s).to eq 'tshark.len'
+      expect(@parsed[:op].to_s).to eq '='
+      expect(@parsed[:value][:term].to_s).to eq '226 '
+      expect(@parsed[:value][:extractors].to_s).to eq 'tshark.frame.time_epoch,tshark.ip.src'
+    end
+  end
+end

data/spec/transformer_spec.rb

@@ -0,0 +1,10 @@
+
+#   after :each do
+#     transformed = @transformer.apply(@parsed)
+#     if transformed.is_a? Array
+#       transformed.map(&:class).uniq.should =~ Array(ResultSet)
+#     else
+#       transformed.should be_a ResultSet
+#     end
+#   end
+

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: plunk
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Ram Mehta
+- Jamil Bou Kheir
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parslet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Human-friendly query language for Elasticsearch
+email:
+- jamil@elbii.com
+- ram.mehta@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/plunk.rb
+- lib/plunk/elasticsearch.rb
+- lib/plunk/parser.rb
+- lib/plunk/result_set.rb
+- lib/plunk/transformer.rb
+- plunk.gemspec
+- spec/elasticsearch_spec.rb
+- spec/parser_spec.rb
+- spec/result_set_spec.rb
+- spec/transformer_spec.rb
+homepage: https://github.com/elbii/plunk
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Elasticsearch query language
+test_files: []