peruse 0.4.0

data/lib/peruse/parser.rb

@@ -0,0 +1,181 @@
+require 'parslet'
+
+module Peruse
+  class Parser < Parslet::Parser
+
+    # BUILDING BLOCKS
+
+    # Single character rules
+    rule(:lparen)     { str('(') >> space? }
+    rule(:rparen)     { str(')') >> space? }
+    rule(:digit)      { match('[0-9]') }
+    rule(:space)      { match('\s').repeat(1) }
+    rule(:space?)     { space.maybe }
+
+    # Numbers
+    rule(:positive_integer) { digit.repeat(1) >> space? }
+    rule(:negative_integer) { str('-') >> positive_integer }
+    rule(:integer)    { negative_integer | positive_integer }
+    rule(:float)      {
+      str('-').maybe >> digit.repeat(1) >> str('.') >> digit.repeat(1) >> space?
+    }
+    rule(:number)     { integer | float }
+
+    # Dates
+    rule(:datetime) {
+      # 1979-05-27T07:32:00Z
+      digit.repeat(4) >> str("-") >> 
+      digit.repeat(2) >> str("-") >> 
+      digit.repeat(2) >> str("T") >> 
+      digit.repeat(2) >> str(":") >> 
+      digit.repeat(2) >> str(":") >> 
+      digit.repeat(2) >> str("Z")
+    }
+
+    # Strings
+    rule(:escaped_special) {
+      str("\\") >> match['0tnr"\\\\']
+    }
+    rule(:string_special) {
+      match['\0\t\n\r"\\\\']
+    }
+    rule(:string) {
+      str('"') >>
+      (escaped_special | string_special.absent? >> any).repeat >>
+      str('"')
+    }
+
+    # Booleans
+    rule(:and_operator) { (str('and') | str('AND') | str('&')) >> space? }
+    rule(:or_operator)  { (str('or') | str('OR') | str('|'))  >> space? }
+    rule(:not_operator) { (str('not') | str('NOT') | str('~')) >> space? }
+
+
+    # COMMANDS
+
+    # Command parts
+    rule(:enclosed_string) { 
+      string | match('[^\s]').repeat
+    }
+    rule(:identifier) { match('[^=\s)(|]').repeat(1) >> match('[^=\s]').repeat }
+    rule(:wildcard)   {
+      (lparen >> wildcard >> rparen) |
+      match('[^=\s|)(]').repeat(1)
+    }
+    rule(:query_value) { string | wildcard | datetime | number }
+    rule(:searchop)   { match['='] }
+    rule(:rhs) {
+      regexp | query_value
+    }
+    rule(:relative_time) {
+      integer.as(:quantity) >>
+      match('s|m|h|d|w').as(:quantifier)
+    }
+    rule(:absolute_time) {
+      datetime.as(:datetime) | enclosed_string.as(:chronic_time)
+    }
+
+    # Field = Value
+    rule(:field_value) {
+      identifier.as(:field) >> space? >>
+      searchop >> space? >>
+      (rhs.as(:value) | subsearch.as(:subsearch))
+    }
+
+    # Value-only
+    rule(:value_only) {
+      query_value.as(:value)
+    }
+
+    # Window
+    rule(:window) {
+      str('window') >>
+      space >>
+      (relative_time | absolute_time).as(:window_start) >>
+      space >> str('to') >> space >>
+      (relative_time | absolute_time).as(:window_end)
+    }
+
+    # Indices
+    rule(:indices) {
+      str('indices') >>
+      space >>
+      (enclosed_string >>
+        (space? >>
+          str(',') >>
+          space? >>
+          enclosed_string).repeat).as(:indices)
+    }
+
+    # Limit
+    rule(:limit) {
+      str('limit') >> space >> positive_integer.as(:limit)
+    }
+
+    # Regexp
+    rule(:regexp) {
+      str('/') >> (str('\/') | match('[^/]')).repeat.as(:regexp) >> str('/')
+    }
+
+    # Last
+    rule(:last) {
+      (str('last') >> space >> relative_time).as(:last)
+    }
+
+    # Subsearch
+    rule(:subsearch) {
+      str('`') >> space? >> nested_search >> str('`')
+    }
+    rule(:nested_search) {
+      peruse_query.as(:initial_query) >> space? >> str('|') >> space? >>
+      match('[^`]').repeat.as(:extractors)
+    }
+
+    # Reference your custom commands here to make them eligible for parsing
+    # NOTE: order matters!
+    rule(:command) {
+      (
+        field_value |
+        window      |
+        last        |
+        limit       |
+        indices     |
+        value_only
+      ).as(:command) >> space?
+    }
+
+
+    # QUERY JOINING
+ 
+    rule(:negated_command) {
+      (not_operator >> command.as(:not)) |
+      command
+    }
+    rule(:primary) { lparen >> or_operation >> rparen | negated_command }
+
+    # borrowed from Parslet's boolean algebra example
+    rule(:negated_and) {
+      (not_operator >> and_operation.as(:not)) |
+      and_operation
+    }
+    rule(:and_operation) { 
+      (primary.as(:left) >> and_operator >> 
+        negated_and.as(:right)).as(:and) | 
+      primary }
+      
+    rule(:negated_or) {
+      (not_operator >> or_operation.as(:not)) |
+      or_operation
+    }
+    rule(:or_operation)  { 
+      (and_operation.as(:left) >> or_operator >> 
+        negated_or.as(:right)).as(:or) | 
+      and_operation }
+
+    rule(:peruse_query) {
+      space? >> or_operation >> space?
+    }
+
+    root(:peruse_query)
+  end
+end

data/lib/peruse/result_set.rb

@@ -0,0 +1,16 @@
+module Peruse
+  class ResultSet
+    attr_accessor :query, :query_string
+
+    def initialize(filter)
+      @query = { query: { filtered: { filter: filter }}}
+    end
+
+    def eval
+      Peruse.elasticsearch_client.search(
+        body: @query,
+        size: Peruse.max_number_of_hits || 10
+      ) if @query
+    end
+  end
+end

data/lib/peruse/transformer.rb

@@ -0,0 +1,103 @@
+require 'parslet'
+require 'chronic'
+
+module Peruse
+  class Transformer < Parslet::Transform
+    # Base
+    rule(command: subtree(:command)) do
+      command
+    end
+
+    # Field = Value
+    rule(
+      field: simple(:field),
+      value: simple(:value)
+    ) do
+      Helper.query_builder(
+        String(field) + ":" + String(value)
+      )
+    end
+
+    # Limit
+    rule(limit: simple(:limit)) do
+      Helper.limit_builder(Integer(limit))
+    end
+
+    # Regexp
+    rule(
+      field: simple(:field),
+      value: {
+        regexp: simple(:regexp)
+      }
+    ) do
+      Helper.regexp_builder(String(field), String(regexp))
+    end
+
+    # Value-only
+    rule(value: simple(:value)) do
+      Helper.query_builder(String(value))
+    end
+
+    # Indices
+    rule(indices: simple(:indices)) do
+      list = String(indices).split(',').collect { |s| s.strip }
+      Helper.indices_builder(list)
+    end
+
+    # Last
+    rule(last: subtree(:last)) do
+      start_time = last
+      end_time = Helper.timestamp_format(Time.now)
+
+      Helper.range_builder(start_time, end_time)
+    end
+
+    # Window
+    rule(
+      window_start: subtree(:window_start),
+      window_end: subtree(:window_end)
+    ) do
+      Helper.range_builder(window_start, window_end)
+    end
+
+    # Time parts
+    rule(datetime: simple(:datetime)) do
+      String(datetime)
+    end
+    rule(
+      quantity: simple(:quantity),
+      quantifier: simple(:quantifier)
+    ) do
+      timestamp = Helper.time_query_to_timestamp(
+        Integer(quantity).abs, # last -1h same as last 1h
+        String(quantifier)
+      )
+
+      Helper.timestamp_format timestamp
+    end
+    rule(chronic_time: simple(:chronic_time)) do
+      Helper.timestamp_format Chronic.parse(chronic_time)
+    end
+
+    # Negate
+    rule(negate: subtree(:not)) do
+      { not: negate }
+    end
+
+    # Command joining
+    rule(:or => {
+      left: subtree(:left),
+      right: subtree(:right)
+    }) do
+      Helper.combine_subtrees(left, right, :or)
+    end
+
+    rule(:and => {
+      left: subtree(:left),
+      right: subtree(:right)
+    }) do
+      Helper.combine_subtrees(left, right, :and)
+    end
+
+  end
+end

data/lib/peruse/utils.rb

@@ -0,0 +1,25 @@
+module Peruse
+  class Utils
+    # nested field matcher
+    def self.extract_values(hash, keys)
+      @vals ||= []
+
+      hash.each_pair do |k, v|
+        if v.is_a? Hash
+          extract_values(v, keys)
+        elsif v.is_a? Array
+          v.flatten!
+          if v.first.is_a? Hash
+            v.each { |el| extract_values(el, keys) }
+          elsif keys.include? k
+            @vals += v
+          end
+        elsif keys.include? k
+          @vals << v
+        end
+      end
+
+      return @vals
+    end
+  end
+end

data/lib/peruse/version.rb

@@ -0,0 +1,3 @@
+module Peruse
+  VERSION = "0.4.0"
+end

data/peruse.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/peruse/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name                    = "peruse"
+  s.version                 = Peruse::VERSION
+  s.required_ruby_version   = ">= 1.9.3"
+  s.add_runtime_dependency  "json", "~> 1.8", ">= 1.8.0"
+  s.add_runtime_dependency  "parslet", "~> 1.5", ">= 1.5.0"
+  s.add_runtime_dependency  "elasticsearch", "~> 1.0", ">= 1.0.0"
+  s.add_runtime_dependency  "activesupport", "~> 4.0", ">= 4.0.0"
+  s.add_runtime_dependency  "chronic", "~> 0.10", ">= 0.10.0"
+  s.add_development_dependency "rspec", "~> 3.1", ">= 3.1.0"
+  s.add_development_dependency "timecop", "~> 0.7", ">= 0.7.1"
+  s.executables             << "peruse"
+  s.summary                 = "Elasticsearch query language"
+  s.description             = "Human-friendly query language for Elasticsearch. Formerly known as plunk."
+  s.authors                 = ["Ram Mehta", "Jamil Bou Kheir"]
+  s.email                   = ["ram.mehta@gmail.com", "jamil@elbii.com"]
+  s.files                   = `git ls-files`.split("\n")
+  s.homepage                = "https://github.com/elbii/peruse"
+  s.license                 = "MIT"
+end

data/spec/basic_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe 'basic searches' do
+  def basic_builder(expected)
+    {
+      query: {
+        filtered: {
+          filter: {
+            query: {
+              query_string: {
+                query: expected
+              }
+            }
+          }
+        }
+      }
+    }
+  end
+
+  it 'should parse bar' do
+    result = Peruse.search 'bar'
+    expect(result).to eq(basic_builder('bar'))
+  end
+
+  it 'should parse bar ' do
+    result = Peruse.search 'bar '
+    expect(result).to eq(basic_builder('bar'))
+  end
+
+  it 'should parse (bar) ' do
+    result = Peruse.search '(bar) '
+    expect(result).to eq(basic_builder('bar'))
+  end
+
+  it 'should parse  bar ' do
+    result = Peruse.search ' bar '
+    expect(result).to eq(basic_builder('bar'))
+  end
+end

data/spec/binstub_spec.rb

@@ -0,0 +1,33 @@
+require 'open3'
+
+describe 'binstub' do
+  it 'should execute a valid query from stdin' do
+    result = ""
+
+    Open3.popen3(
+      "bin/peruse",
+      "-h localhost,127.0.0.1",
+      "-s 1",
+      "-r",
+      "-d",
+      "-t timestamp"
+    ) do |stdin, stdout, stderr|
+      stdin.puts "last 1w"
+      stdin.close
+      result = stdout.read.chomp
+    end
+      
+    expect(result).to be_present
+  end
+
+  it 'should not allow invalid options' do
+    exit_status = -1 
+
+    Open3.popen3("bin/peruse", "-z") do |stdin, stdout, stderr, thread|
+      exit_status = thread.value.exitstatus
+    end
+
+
+    expect(exit_status).to eq 1
+  end
+end

data/spec/boolean_spec.rb

@@ -0,0 +1,112 @@
+require 'spec_helper'
+
+describe 'boolean searches' do
+
+  it 'should parse (foo OR bar)' do
+    result = Peruse.search '(foo OR bar)'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        Peruse::Helper.query_builder('foo'),
+        Peruse::Helper.query_builder('bar')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse (foo | bar)' do
+    result = Peruse.search '(foo | bar)'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        Peruse::Helper.query_builder('foo'),
+        Peruse::Helper.query_builder('bar')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse (foo OR (bar AND baz))' do
+    result = Peruse.search '(foo OR (bar AND baz))'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        Peruse::Helper.query_builder('foo'),
+        {
+          and: [
+            Peruse::Helper.query_builder('bar'),
+            Peruse::Helper.query_builder('baz')
+          ]
+        }
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse foo=bar & baz=fez & fad=bad' do
+    result = Peruse.search 'foo=bar & baz=fez & fad=bad'
+    expected = Peruse::Helper.filter_builder({
+      and: [
+        Peruse::Helper.query_builder('foo:bar'),
+        Peruse::Helper.query_builder('baz:fez'),
+        Peruse::Helper.query_builder('fad:bad')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse foo=bar | foo=baz | fez=baz' do
+    result = Peruse.search 'foo=bar | foo=baz | fez=baz'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        Peruse::Helper.query_builder('foo:bar'),
+        Peruse::Helper.query_builder('foo:baz'),
+        Peruse::Helper.query_builder('fez:baz')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse (foo=bar OR foo=bar)' do
+    result = Peruse.search '(foo=bar OR foo=bar)'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        Peruse::Helper.query_builder('foo:bar'),
+        Peruse::Helper.query_builder('foo:bar')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse foo=bar OR baz=fez' do
+    result = Peruse.search 'foo=bar OR baz=fez'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        Peruse::Helper.query_builder('foo:bar'),
+        Peruse::Helper.query_builder('baz:fez')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse (foo=bar AND baz=fez) OR ham=cheese' do
+    result = Peruse.search '(foo=bar AND baz=fez) OR ham=cheese'
+    expected = Peruse::Helper.filter_builder({
+      or: [
+        {
+          and: [
+            Peruse::Helper.query_builder('foo:bar'),
+            Peruse::Helper.query_builder('baz:fez')
+          ]
+        },
+        Peruse::Helper.query_builder('ham:cheese')
+      ]
+    })
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse NOT foo=bar' do
+    result = Peruse.search 'NOT foo=bar'
+    expected = Peruse::Helper.filter_builder({
+      not: Peruse::Helper.query_builder('foo:bar')
+    })
+    expect(result).to eq(expected)
+  end
+end