plunk 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2166c154fb85131ea652ccc4409725b29b4dc250
-  data.tar.gz: 75503b3934ea91d1ba81ff934684ccd724af7a84
+  metadata.gz: f9783fb61da735628ac2266ff16fd4234d146bed
+  data.tar.gz: 8fa8d01bffed0cc5351afef868a2e22caa3ddf88
 SHA512:
-  metadata.gz: bbeff1a1e3c8e36d0857662715b2b8c9392d6543ff7de11527f6748852feba47f449bb1e66c9a125918884c4d95bfa2d3cb952370c77cb87d9f8e1a3ae00f91f
-  data.tar.gz: 1d281d2a627594100c9ed502a96e82e01192248585cba14a8de91174fb5c8381145dd8d8a192a9a7259b0653eed029629fc2642593011fee14c6c7bee4b3aa14
+  metadata.gz: aaf0493c53bfee8441676b7e72697fb72d47e32b5e6abd8e505e6ef89483a7976bc6ec3a5fb5b34b5959fff0a4d5074cb836b9acde565417737589c704c0fb54
+  data.tar.gz: 3767a5c241aa94a9c55faef62742f6e57862377cb2e662cc0c58cd7a952dd1ac5716f3ddea6a894adf73f806c0db5481c95e4f99c1804033a59d9f1875a7a2bf

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    plunk (0.3.1)
+    plunk (0.3.2)
       activesupport (~> 4.0, >= 4.0.0)
+      chronic (~> 0.10, >= 0.10.0)
       elasticsearch (~> 0.4, >= 0.4.3)
       json (~> 1.8, >= 1.8.0)
       parslet (~> 1.5, >= 1.5.0)
@@ -18,6 +19,7 @@ GEM
       tzinfo (~> 0.3.37)
     atomic (1.1.14)
     blankslate (2.1.2.4)
+    chronic (0.10.2)
     diff-lcs (1.2.5)
     elasticsearch (0.4.3)
       elasticsearch-api (= 0.4.3)

data/README.md

@@ -5,43 +5,82 @@ Human-friendly query language for Elasticsearch
 
 ## About
 
-Plunk is a ruby gem to take a human-friendly search command and translate it
-to something Elasticsearch understands. Currently it only supports a few
-commands, but the framework is in place
+Plunk is a ruby gem to take a human-friendly, one-line search command and
+translate it to full-fledged JSON to send to Elasticsearch. Currently it only
+supports a few commands, but the goal is to support a large subset of what
+Elasticsearch offers.
 
 ## Installation
 ```
 gem install plunk
 ```
 
+Plunk uses [Parslet](https://github.com/kschiess/parslet) to first parse your
+query, and then [Elasticsearch's official ruby library](https://github.com/elasticsearch/elasticsearch-ruby)
+to send it to Elasticsearch.
+
 ## Usage
 ```ruby
 require 'plunk'
 
-# configure is required
-# elasticsearch_options accepts the same params as Elasticsearch::Client
+# 
+# Configuration is required before using Plunk
+# 
+# Elasticsearch_options accepts the same params as Elasticsearch::Client
+# from the elasticsearch-ruby library
 Plunk.configure do |config|
   config.elasticsearch_options = { host: 'localhost' }
 end
 
-# all documents of type syslog from the last week
-Plunk.search 'last 1w _type = syslog'
+# Restrict timeframe to last 1 week and match documents with _type=syslog
+# s = seconds
+# m = minutes
+# h = hours
+# d = days
+# w = weeks
+# All times in Plunk are converted to UTC
+Plunk.search 'last 1w AND _type = syslog'
+
+# The ```window``` command can also be used to filter by time
+Plunk.search 'window -2d to -1d'
+
+# Plunk tries to parse the date with Chronic, so this works too. Note the
+# double quotes around the time string. This is needed if it contains a space.
+Plunk.search 'window "last monday" to "last thursday"'
+
+# Of course, absolute dates are supported as well. Date format is American style
+# e.g. MM/DD/YY
+Plunk.search 'window 3/14/12 to 3/15/12'
+
+# Use double quotes to wrap space-containing strings
+Plunk.search 'http.header = "UserAgent: Mozilla/5.0"'
+
+# Commands are joined using parenthesized booleans
+Plunk.search '(last 1h AND severity = 5) OR (last 1w AND severity = 3)'
 
-# nested search. first runs the '_type=access_logs' search, extracts the values
-# for fields user.name, user.nickname, and user.email, then ORs these together
-# as the query for the outer search.
-Plunk.search 'user = `_type=access_logs | user.name, user.nickname, user.email`'
+# "AND" is aliased to "and" and "&". Similarly, "OR" is aliased to "or" and "|".
+# The following queries are identical to one above
+Plunk.search '(last 1h and severity = 5) or (last 1w and severity = 3)'
+Plunk.search '(last 1h & severity = 5) | (last 1w & severity = 3)'
 
-# booleans are supported
-Plunk.search 'foo.field = (bar OR baz)'
+# Use the NOT keyword to negate the following command or boolean chain
+Plunk.search 'NOT message = Error'
+
+# Like AND and OR, "NOT" is aliased to "not" and "~"
+Plunk.search 'not message = Error'
+Plunk.search '~ message = Error'
+
+# Regexp is supported as well
+Plunk.search 'http.headers = /.*User-Agent: Mozilla.*/ OR http.headers = /.*application\/json.*/'
 ```
 
 
 ## Translation
 
-Plunk takes your command and translates
+Under the hood, Plunk takes your query and translates it to
+Elasticsearch-compatible JSON. For example,
 
-```last 24h _type=syslog```
+```last 24h & _type=syslog```
 
 gets translated to:
 
@@ -49,11 +88,6 @@ gets translated to:
 {
   "query": {
     "filtered": {
-      "query": {
-        "query_string": {
-          "query": "_type:syslog"
-        }
-      },
       "filter": {
         "and": [
           {
@@ -63,6 +97,13 @@ gets translated to:
                 "lte": "2013-08-24T05:43:13.770Z"
               }
             }
+          },
+          {
+            "query": {
+              "query_string": {
+                "query": "_type:syslog"
+              }
+            }
           }
         ]
       }
@@ -70,3 +111,6 @@ gets translated to:
   }
 }
 ```
+
+In general, commands are combined into a single filter using Elasticsearch's,
+```and```, ```or```, and ```not``` filters.

data/lib/plunk/helper.rb

@@ -30,17 +30,34 @@ module Plunk
       }
     end
 
+    def self.limit_builder(limit)
+      {
+        limit: {
+          value: limit
+        }
+      }
+    end
+
     def self.range_builder(range_min, range_max)
       {
         range: {
           Plunk.timestamp_field => {
             gte: range_min,
-            gte: range_max
+            lte: range_max
           }
         }
       }
     end
 
+    def self.regexp_builder(field, regexp, flags=nil)
+      {
+        regexp: {
+          field => regexp,
+          flags: flags || 'ALL'
+        }
+      }
+    end
+
     def self.time_query_to_timestamp(int_quantity, quantifier)
       case quantifier
       when 's'

data/lib/plunk/parser.rb

@@ -13,7 +13,9 @@ module Plunk
     rule(:space?)     { space.maybe }
 
     # Numbers
-    rule(:integer)    { str('-').maybe >> digit.repeat(1) >> space? }
+    rule(:positive_integer) { digit.repeat(1) >> space? }
+    rule(:negative_integer) { str('-') >> positive_integer }
+    rule(:integer)    { negative_integer | positive_integer }
     rule(:float)      {
       str('-').maybe >> digit.repeat(1) >> str('.') >> digit.repeat(1) >> space?
     }
@@ -62,6 +64,16 @@ module Plunk
     rule(:rhs) {
       regexp | query_value
     }
+    rule(:chronic_time) {
+      string | match('[^\s]').repeat
+    }
+    rule(:relative_time) {
+      integer.as(:quantity) >>
+      match('s|m|h|d|w').as(:quantifier)
+    }
+    rule(:absolute_time) {
+      datetime.as(:datetime) | chronic_time.as(:chronic_time)
+    }
 
     # Field = Value
     rule(:field_value) {
@@ -72,20 +84,31 @@ module Plunk
 
     # Value-only
     rule(:value_only) {
-      rhs.as(:value)
+      query_value.as(:value)
+    }
+
+    # Window
+    rule(:window) {
+      str('window') >>
+      space >>
+      (relative_time | absolute_time).as(:window_start) >>
+      space >> str('to') >> space >>
+      (relative_time | absolute_time).as(:window_end)
+    }
+
+    # Limit
+    rule(:limit) {
+      str('limit') >> space >> positive_integer.as(:limit)
     }
 
     # Regexp
     rule(:regexp) {
-      str('/') >> (str('\/') | match('[^/]')).repeat >> str('/')
+      str('/') >> (str('\/') | match('[^/]')).repeat.as(:regexp) >> str('/')
     }
 
     # Last
     rule(:last) {
-      str('last') >>
-      space >>
-      integer.as(:quantity) >>
-      match('s|m|h|d|w').as(:quantifier)
+      (str('last') >> space >> relative_time).as(:last)
     }
 
     # Subsearch
@@ -101,7 +124,9 @@ module Plunk
     # NOTE: order matters!
     rule(:command) {
       (
+        window      |
         last        |
+        limit       |
         field_value |
         value_only
       ).as(:command) >> space?

data/lib/plunk/transformer.rb

@@ -1,41 +1,84 @@
 require 'parslet'
+require 'chronic'
 
 module Plunk
   class Transformer < Parslet::Transform
+    # Base
+    rule(command: subtree(:command)) do
+      command
+    end
+
     # Field = Value
-    rule(command: {
+    rule(
       field: simple(:field),
       value: simple(:value)
-    }) do
+    ) do
       Helper.query_builder(
         String(field) + ":" + String(value)
       )
     end
 
+    # Limit
+    rule(limit: simple(:limit)) do
+      Helper.limit_builder(Integer(limit))
+    end
+
+    # Regexp
+    rule(
+      field: simple(:field),
+      value: {
+        regexp: simple(:regexp)
+      }
+    ) do
+      Helper.regexp_builder(String(field), String(regexp))
+    end
+
     # Value-only
-    rule(command: { value: simple(:value) }) do
+    rule(value: simple(:value)) do
       Helper.query_builder(String(value))
     end
 
-    rule(command: {
+    # Last
+    rule(last: subtree(:last)) do
+      start_time = last
+      end_time = Helper.timestamp_format(Time.now)
+
+      Helper.range_builder(start_time, end_time)
+    end
+
+    # Window
+    rule(
+      window_start: subtree(:window_start),
+      window_end: subtree(:window_end)
+    ) do
+      Helper.range_builder(window_start, window_end)
+    end
+
+    # Time parts
+    rule(datetime: simple(:datetime)) do
+      String(datetime)
+    end
+    rule(
       quantity: simple(:quantity),
       quantifier: simple(:quantifier)
-    }) do
-      start_timestamp = Helper.time_query_to_timestamp(
-        Integer(quantity),
+    ) do
+      timestamp = Helper.time_query_to_timestamp(
+        Integer(quantity).abs, # last -1h same as last 1h
         String(quantifier)
       )
 
-      start_time = Helper.timestamp_format start_timestamp
-      end_time = Helper.timestamp_format(Time.now)
-
-      Helper.range_builder(start_time, end_time)
+      Helper.timestamp_format timestamp
+    end
+    rule(chronic_time: simple(:chronic_time)) do
+      Helper.timestamp_format Chronic.parse(chronic_time)
     end
 
-    rule(:negate => subtree(:not)) do
+    # Negate
+    rule(negate: subtree(:not)) do
       { not: negate }
     end
 
+    # Command joining
     rule(:or => {
       left: subtree(:left),
       right: subtree(:right)

data/plunk.gemspec

@@ -1,10 +1,11 @@
 Gem::Specification.new do |s|
   s.name                    = "plunk"
-  s.version                 = "0.3.2"
+  s.version                 = "0.3.3"
   s.add_runtime_dependency  "json", "~> 1.8", ">= 1.8.0"
   s.add_runtime_dependency  "parslet", "~> 1.5", ">= 1.5.0"
   s.add_runtime_dependency  "elasticsearch", "~> 0.4", ">= 0.4.3"
   s.add_runtime_dependency  "activesupport", "~> 4.0", ">= 4.0.0"
+  s.add_runtime_dependency  "chronic", "~> 0.10", ">= 0.10.0"
   s.add_development_dependency "rspec", "~> 2.0", ">= 2.14.1"
   s.add_development_dependency "timecop", "~> 0.7", ">= 0.7.1"
   s.summary                 = "Elasticsearch query language"

data/spec/limit_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe 'the limit command' do
+  it 'should parse limit 1' do
+    result = Plunk.search 'limit 1'
+    expected = Plunk::Helper.filter_builder(
+      Plunk::Helper.limit_builder(1)
+    )
+    expect(result).to eq(expected)
+  end
+end

data/spec/regexp_spec.rb

@@ -5,7 +5,10 @@ describe 'regexp searches' do
   it 'should parse foo=/blah foo/' do
     result = Plunk.search 'foo=/blah foo/'
     expected = Plunk::Helper.filter_builder(
-      Plunk::Helper.query_builder('foo:/blah foo/')
+      Plunk::Helper.regexp_builder(
+        'foo',
+        'blah foo'
+      )
     )
     expect(result).to eq(expected)
   end
@@ -13,7 +16,10 @@ describe 'regexp searches' do
   it 'should parse foo=/blah\/ foo/' do
     result = Plunk.search 'foo=/blah\/ foo/'
     expected = Plunk::Helper.filter_builder(
-      Plunk::Helper.query_builder('foo:/blah\/ foo/')
+      Plunk::Helper.regexp_builder(
+        'foo',
+        'blah\/ foo'
+      )
     )
     expect(result).to eq(expected)
   end
@@ -21,15 +27,21 @@ describe 'regexp searches' do
   it 'should parse foo=/blah\. foo/' do
     result = Plunk.search 'foo=/blah\. foo/'
     expected = Plunk::Helper.filter_builder(
-      Plunk::Helper.query_builder('foo:/blah\. foo/')
+      Plunk::Helper.regexp_builder(
+        'foo',
+        'blah\. foo'
+      )
     )
     expect(result).to eq(expected)
   end
 
-  it 'should parse /.*User\-Agent\: Microsoft\-WebDAV.*/' do
-    result = Plunk.search '/.*User\-Agent\: Microsoft\-WebDAV.*/'
+  it 'should parse http.headers=/.*User\-Agent\: Microsoft\-WebDAV.*/' do
+    result = Plunk.search 'http.headers=/.*User\-Agent\: Microsoft\-WebDAV.*/'
     expected = Plunk::Helper.filter_builder(
-      Plunk::Helper.query_builder('/.*User\-Agent\: Microsoft\-WebDAV.*/')
+      Plunk::Helper.regexp_builder(
+        'http.headers',
+        '.*User\-Agent\: Microsoft\-WebDAV.*'
+      )
     )
     expect(result).to eq(expected)
   end

data/spec/window_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'shared/time_stubs'
+require 'shared/plunk_stubs'
+require 'chronic'
+
+describe 'the window command' do
+  include_context 'time stubs'
+  include_context 'plunk stubs'
+
+  it 'should parse window 3/11/13 to 3/12/13' do
+    result = Plunk.search 'window 3/11/13 to 3/12/13'
+    expected = Plunk::Helper.filter_builder(
+      Plunk::Helper.range_builder(
+        Plunk::Helper.timestamp_format(Chronic.parse('3/11/13')),
+        Plunk::Helper.timestamp_format(Chronic.parse('3/12/13'))
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse window "last monday" to "last thursday"' do
+    result = Plunk.search 'window "last monday" to "last thursday"'
+    expected = Plunk::Helper.filter_builder(
+      Plunk::Helper.range_builder(
+        Plunk::Helper.timestamp_format(Chronic.parse('last monday')),
+        Plunk::Helper.timestamp_format(Chronic.parse('last thursday'))
+      )
+    )
+    expect(result).to eq(expected)
+  end
+
+  it 'should parse window -60m to -30m' do
+    result = Plunk.search 'window -60m to -30m'
+    expected = Plunk::Helper.filter_builder(
+      Plunk::Helper.range_builder(
+        Plunk::Helper.timestamp_format(@time - 60.minutes),
+        Plunk::Helper.timestamp_format(@time - 30.minutes)
+      )
+    )
+    expect(result).to eq(expected)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: plunk
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Ram Mehta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-20 00:00:00.000000000 Z
+date: 2014-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -92,6 +92,26 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: chronic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -161,12 +181,14 @@ files:
 - spec/chained_search_spec.rb
 - spec/field_value_spec.rb
 - spec/last_spec.rb
+- spec/limit_spec.rb
 - spec/nested_search_spec.rb
 - spec/regexp_spec.rb
 - spec/shared/dummy_client.rb
 - spec/shared/plunk_stubs.rb
 - spec/shared/time_stubs.rb
 - spec/spec_helper.rb
+- spec/window_spec.rb
 homepage: https://github.com/elbii/plunk
 licenses:
 - MIT