peruse 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 15b1475221dbde1e53dcce6955d47fc167274e62
+  data.tar.gz: d28ee6470fa23b902465727c41ea6aa642a3a5dd
+SHA512:
+  metadata.gz: f1072f67995b3698514c4002d2ada8cf34c52ee16eae3331160a2ba0cd3eaf1f2eb1b51f3bedc78f0f96f0b816400eb37fc19e0db16e249d6b31d17859fee92d
+  data.tar.gz: df29c59ec1b7915342e57cebf3c40a87b92b86dd7700f46d527f86cfe4e15551dbaa45fa02c635825979a22c2f48b7b53a35197b657b7ba16cd8d1b3dab2a261

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/.ruby-version

@@ -0,0 +1 @@
+2.1

data/AUTHORS

@@ -0,0 +1,2 @@
+Jamil Bou Kheir <jamil@elbii.com>
+Ram Mehta <ram.mehta@gmail.com>

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,63 @@
+PATH
+  remote: .
+  specs:
+    plunk (0.3.11)
+      activesupport (~> 4.0, >= 4.0.0)
+      chronic (~> 0.10, >= 0.10.0)
+      elasticsearch (~> 1.0, >= 1.0.0)
+      json (~> 1.8, >= 1.8.0)
+      parslet (~> 1.5, >= 1.5.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    blankslate (2.1.2.4)
+    chronic (0.10.2)
+    diff-lcs (1.2.5)
+    elasticsearch (1.0.4)
+      elasticsearch-api (= 1.0.4)
+      elasticsearch-transport (= 1.0.4)
+    elasticsearch-api (1.0.4)
+      multi_json
+    elasticsearch-transport (1.0.4)
+      faraday
+      multi_json
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    i18n (0.6.11)
+    json (1.8.1)
+    minitest (5.4.0)
+    multi_json (1.10.1)
+    multipart-post (2.0.0)
+    parslet (1.6.1)
+      blankslate (~> 2.0)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.0)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.0)
+    thread_safe (0.3.4)
+    timecop (0.7.1)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  plunk!
+  rspec (~> 3.1, >= 3.1.0)
+  timecop (~> 0.7, >= 0.7.1)

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Elbii
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,118 @@
+Note: Plunk has been renamed to Peruse
+
+Peruse
+======
+
+Human-friendly query language for Elasticsearch
+
+## About
+
+Peruse is a ruby gem to take a human-friendly, one-line search command and
+translate it to full-fledged JSON to send to Elasticsearch. Currently it only
+supports a few commands, but the goal is to support a large subset of what
+Elasticsearch offers.
+
+## Installation
+```
+gem install peruse
+```
+
+Peruse uses [Parslet](https://github.com/kschiess/parslet) to first parse your
+query, and then [Elasticsearch's official ruby library](https://github.com/elasticsearch/elasticsearch-ruby)
+to send it to Elasticsearch.
+
+## Usage
+```ruby
+require 'peruse'
+
+# 
+# Configuration is required before using Peruse
+# 
+# Elasticsearch_options accepts the same params as Elasticsearch::Client
+# from the elasticsearch-ruby library
+Peruse.configure do |config|
+  config.elasticsearch_options = { host: 'localhost' }
+end
+
+# Restrict timeframe to last 1 week and match documents with _type=syslog
+# s = seconds
+# m = minutes
+# h = hours
+# d = days
+# w = weeks
+# All times in Peruse are converted to UTC
+Peruse.search 'last 1w AND _type = syslog'
+
+# The ```window``` command can also be used to filter by time
+Peruse.search 'window -2d to -1d'
+
+# Peruse tries to parse the date with Chronic, so this works too. Note the
+# double quotes around the time string. This is needed if it contains a space.
+Peruse.search 'window "last monday" to "last thursday"'
+
+# Of course, absolute dates are supported as well. Date format is American style
+# e.g. MM/DD/YY
+Peruse.search 'window 3/14/12 to 3/15/12'
+
+# Use double quotes to wrap space-containing strings
+Peruse.search 'http.header = "UserAgent: Mozilla/5.0"'
+
+# Commands are joined using parenthesized booleans
+Peruse.search '(last 1h AND severity = 5) OR (last 1w AND severity = 3)'
+
+# "AND" is aliased to "and" and "&". Similarly, "OR" is aliased to "or" and "|".
+# The following queries are identical to one above
+Peruse.search '(last 1h and severity = 5) or (last 1w and severity = 3)'
+Peruse.search '(last 1h & severity = 5) | (last 1w & severity = 3)'
+
+# Use the NOT keyword to negate the following command or boolean chain
+Peruse.search 'NOT message = Error'
+
+# Like AND and OR, "NOT" is aliased to "not" and "~"
+Peruse.search 'not message = Error'
+Peruse.search '~ message = Error'
+
+# Regexp is supported as well
+Peruse.search 'http.headers = /.*User-Agent: Mozilla.*/ OR http.headers = /.*application\/json.*/'
+```
+
+
+## Translation
+
+Under the hood, Peruse takes your query and translates it to
+Elasticsearch-compatible JSON. For example,
+
+```last 24h & _type=syslog```
+
+gets translated to:
+
+```json
+{
+  "query": {
+    "filtered": {
+      "filter": {
+        "and": [
+          {
+            "range": {
+              "timestamp": {
+                "gte": "2013-08-23T05:43:13.770Z",
+                "lte": "2013-08-24T05:43:13.770Z"
+              }
+            }
+          },
+          {
+            "query": {
+              "query_string": {
+                "query": "_type:syslog"
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+```
+
+In general, commands are combined into a single filter using Elasticsearch's,
+```and```, ```or```, and ```not``` filters.

data/Rakefile

@@ -0,0 +1,5 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/peruse

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << './lib'
+require 'peruse'
+require 'optparse'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: peruse [options]"
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on(
+    "-h",
+    "--host HOST",
+    "comma-separated list of Elasticsearch hosts to use"
+  ) do |option|
+    options[:host] = option
+  end
+
+  opts.on(
+    "-p",
+    "--parse-only",
+    "parse but don't execute query, returning ES-compatible JSON"
+  ) do |option|
+    options[:parse_only] = option
+  end
+
+  opts.on(
+    "-s",
+    "--size SIZE",
+    "max number of hits to return"
+  ) do |option|
+    options[:size] = option
+  end
+  
+  opts.on(
+    "-r",
+    "--randomize-hosts",
+    "randomize hosts used for each search"
+  ) do |option|
+    options[:randomize_hosts] = option
+  end
+
+  opts.on(
+    "-t",
+    "--timestamp-field FIELD",
+    "timestamp field to use for timerange searches"
+  ) do |option|
+    options[:timestamp_field] = option
+  end
+
+  opts.on(
+    "-d",
+    "--debug",
+    "turn on debugging output"
+  ) do |option|
+    options[:debug] = option
+  end
+
+end.parse!
+
+Peruse.configure do |c|
+  c.parse_only = options[:parse_only]
+  c.max_number_of_hits = options[:size].to_i if options[:size]
+  c.timestamp_field =
+    options[:timestamp_field].strip if options[:timstamp_field]
+  c.logger = Logger.new(STDOUT) if options[:debug]
+
+  c.elasticsearch_client = Elasticsearch::Client.new(
+    host: options[:host].split(',').collect! { |h| h.strip },
+    randomize_hosts: options[:randomize_hosts]
+  ) unless c.parse_only
+end
+
+puts Peruse.search($stdin.read).to_json

data/examples/simple.json

@@ -0,0 +1,25 @@
+{
+   "query":{
+      "filtered":{
+         "filter":{
+            "and":[
+               {
+                  "range":{
+                     "timestamp":{
+                        "gte":"2014-04-01T16:00:00.000+00:00",
+                        "lte":"2014-04-07T16:00:00.000+00:00"
+                     }
+                  }
+               },
+               {
+                  "query":{
+                     "query_string":{
+                        "query":"_type:syslog"
+                     }
+                  }
+               }
+            ]
+         }
+      }
+   }
+}

data/examples/simple.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH << './lib'
+require './lib/peruse'
+
+Peruse.configure do |c|
+  c.elasticsearch_options = { host: 'localhost' }
+  c.timestamp_field = :timestamp
+end
+
+query = 'window "last monday" to "last tuesday" & _type = syslog' 
+puts Peruse.search query

data/lib/peruse.rb

@@ -0,0 +1,53 @@
+require 'elasticsearch'
+
+require 'peruse/helper'
+require 'peruse/utils'
+require 'peruse/parser'
+require 'peruse/transformer'
+require 'peruse/result_set'
+
+module Peruse
+  class << self
+    attr_accessor :elasticsearch_options, :elasticsearch_client,
+      :parser, :transformer, :max_number_of_hits, :timestamp_field, :logger,
+      :parse_only
+  end
+
+  def self.configure(&block)
+    class_eval(&block)
+    self.timestamp_field ||= :timestamp
+    initialize_parser
+    initialize_transformer
+    initialize_elasticsearch unless self.parse_only
+  end
+
+  def self.initialize_elasticsearch
+    self.elasticsearch_client ||= Elasticsearch::Client.new(elasticsearch_options)
+  end
+
+  def self.initialize_parser
+    self.parser ||= Parser.new
+  end
+
+  def self.initialize_transformer
+    self.transformer ||= Transformer.new
+  end
+
+  def self.search(query_string)
+    parsed = parser.parse query_string
+    transformed = transformer.apply parsed
+
+    if self.logger
+      self.logger.debug "Query String: #{query_string}"
+      self.logger.debug "Parsed Output: #{transformed}" 
+    end
+
+    result_set = ResultSet.new(transformed)
+
+    if self.parse_only
+      result_set.query
+    else
+      result_set.eval
+    end
+  end
+end

data/lib/peruse/helper.rb

@@ -0,0 +1,90 @@
+require 'active_support/core_ext'
+
+module Peruse
+  class Helper
+    def self.combine_subtrees(left, right, op)
+      if right[op]
+        { op => [left] + right[op] }
+      else
+        { op => [left, right] }
+      end
+    end
+
+    def self.query_builder(query_string)
+      {
+        query: {
+          query_string: {
+            query: query_string
+          }
+        }
+      }
+    end
+
+    def self.filter_builder(filter)
+      {
+        query: {
+          filtered: {
+            filter: filter
+          }
+        }
+      }
+    end
+
+    def self.limit_builder(limit)
+      {
+        limit: {
+          value: limit
+        }
+      }
+    end
+
+    def self.range_builder(range_min, range_max)
+      {
+        range: {
+          Peruse.timestamp_field => {
+            gte: range_min,
+            lte: range_max
+          }
+        }
+      }
+    end
+
+    def self.regexp_builder(field, regexp, flags=nil)
+      {
+        regexp: {
+          field => {
+            value: regexp,
+            flags: flags || 'ALL'
+          }
+        }
+      }
+    end
+
+    def self.indices_builder(list)
+      {
+        indices: {
+          indices: list
+        }
+      }
+    end
+
+    def self.time_query_to_timestamp(int_quantity, quantifier)
+      case quantifier
+      when 's'
+        int_quantity.seconds.ago
+      when 'm'
+        int_quantity.minutes.ago
+      when 'h'
+        int_quantity.hours.ago
+      when 'd'
+        int_quantity.days.ago
+      when 'w'
+        int_quantity.weeks.ago
+      end
+    end
+
+    def self.timestamp_format(time)
+      time.utc.to_datetime.iso8601(3)
+    end
+  end
+end