plunk 0.1.7 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +13 -5
- data/README.md +32 -2
- data/lib/plunk.rb +36 -1
- data/lib/plunk/parser.rb +116 -115
- data/lib/plunk/result_set.rb +32 -26
- data/lib/plunk/transformer.rb +129 -129
- data/lib/plunk/utils.rb +25 -0
- data/plunk.gemspec +2 -2
- data/spec/spec_helper.rb +8 -10
- metadata +3 -4
- data/lib/plunk/elasticsearch.rb +0 -95
- data/spec/elasticseach_spec.rb +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9c820c78844ed02012137a7d2d0c347ae6a6ac8e
|
4
|
+
data.tar.gz: 1e6ae6df764d20dc39160a7d1706b3dcfdbd910a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ade712790a1a1088205e3d4dfd4020b5bbe7db8327ea87532bb2f4a02dd6220f03b161da675ffbf16a8f0dc34793bf361f53d0081d8668eda2d478cdc7ddfad5
|
7
|
+
data.tar.gz: 26f74070375857a46edcead703bc240b17bef95711189248b89838fe303bcdb011ef29dfd7e816de9211de7750c8a08625c642aa27d47076084a34c9ed73b263
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
plunk (0.
|
4
|
+
plunk (0.2.0)
|
5
5
|
activesupport
|
6
|
+
elasticsearch
|
6
7
|
json
|
7
8
|
parslet
|
8
|
-
rest-client
|
9
9
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
@@ -19,15 +19,23 @@ GEM
|
|
19
19
|
atomic (1.1.14)
|
20
20
|
blankslate (2.1.2.4)
|
21
21
|
diff-lcs (1.2.5)
|
22
|
+
elasticsearch (0.4.3)
|
23
|
+
elasticsearch-api (= 0.4.3)
|
24
|
+
elasticsearch-transport (= 0.4.3)
|
25
|
+
elasticsearch-api (0.4.3)
|
26
|
+
multi_json
|
27
|
+
elasticsearch-transport (0.4.3)
|
28
|
+
faraday
|
29
|
+
multi_json
|
30
|
+
faraday (0.8.8)
|
31
|
+
multipart-post (~> 1.2.0)
|
22
32
|
i18n (0.6.9)
|
23
33
|
json (1.8.1)
|
24
|
-
mime-types (2.0)
|
25
34
|
minitest (4.7.5)
|
26
35
|
multi_json (1.8.2)
|
36
|
+
multipart-post (1.2.0)
|
27
37
|
parslet (1.5.0)
|
28
38
|
blankslate (~> 2.0)
|
29
|
-
rest-client (1.6.7)
|
30
|
-
mime-types (>= 1.16)
|
31
39
|
rspec (2.14.1)
|
32
40
|
rspec-core (~> 2.14.0)
|
33
41
|
rspec-expectations (~> 2.14.0)
|
data/README.md
CHANGED
@@ -1,9 +1,39 @@
|
|
1
|
-
|
1
|
+
Plunk
|
2
2
|
=====
|
3
3
|
|
4
4
|
Human-friendly query language for Elasticsearch
|
5
5
|
|
6
|
-
|
6
|
+
Currently assumes you're using Logstash to
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
```
|
10
|
+
gem install plunk
|
11
|
+
```
|
12
|
+
|
13
|
+
## Usage
|
14
|
+
```ruby
|
15
|
+
require 'plunk'
|
16
|
+
|
17
|
+
# configure is required
|
18
|
+
# elasticsearch_options accepts the same params as Elasticsearch::Client
|
19
|
+
Plunk.configure do |config|
|
20
|
+
config.elasticsearch_options = { host: 'localhost' }
|
21
|
+
end
|
22
|
+
|
23
|
+
# all documents of type syslog from the last week
|
24
|
+
Plunk.search 'last 1w _type = syslog'
|
25
|
+
|
26
|
+
# nested search. first runs the '_type=access_logs' search, extracts the values
|
27
|
+
# for fields user.name, user.nickname, and user.email, then ORs these together
|
28
|
+
# as the query for the outer search.
|
29
|
+
Plunk.search 'user = `_type=access_logs | user.name, user.nickname, user.email`'
|
30
|
+
|
31
|
+
# booleans are supported
|
32
|
+
Plunk.search 'foo.field = (bar OR baz)'
|
33
|
+
```
|
34
|
+
|
35
|
+
|
36
|
+
## Translation:
|
7
37
|
|
8
38
|
```last 24h _type=syslog```
|
9
39
|
|
data/lib/plunk.rb
CHANGED
@@ -1,3 +1,38 @@
|
|
1
|
-
|
1
|
+
require 'elasticsearch'
|
2
2
|
|
3
|
+
require 'plunk/utils'
|
4
|
+
require 'plunk/parser'
|
5
|
+
require 'plunk/transformer'
|
6
|
+
require 'plunk/result_set'
|
7
|
+
|
8
|
+
module Plunk
|
9
|
+
class << self
|
10
|
+
attr_accessor :elasticsearch_options, :elasticsearch_client,
|
11
|
+
:parser, :transformer
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.configure(&block)
|
15
|
+
class_eval(&block)
|
16
|
+
initialize_elasticsearch
|
17
|
+
initialize_parser
|
18
|
+
initialize_transformer
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.initialize_elasticsearch
|
22
|
+
self.elasticsearch_client ||= Elasticsearch::Client.new elasticsearch_options
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.initialize_parser
|
26
|
+
self.parser ||= Parser.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.initialize_transformer
|
30
|
+
self.transformer ||= Transformer.new
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.search(query_string)
|
34
|
+
parsed = parser.parse query_string
|
35
|
+
result_set = transformer.apply parsed
|
36
|
+
result_set.eval
|
37
|
+
end
|
3
38
|
end
|
data/lib/plunk/parser.rb
CHANGED
@@ -1,119 +1,120 @@
|
|
1
1
|
require 'parslet'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
module Plunk
|
4
|
+
class Parser < Parslet::Parser
|
5
|
+
|
6
|
+
def parenthesized(atom)
|
7
|
+
lparen >> atom >> rparen
|
8
|
+
end
|
9
|
+
|
10
|
+
# Single character rules
|
11
|
+
rule(:lparen) { str('(') >> space? }
|
12
|
+
rule(:rparen) { str(')') >> space? }
|
13
|
+
rule(:comma) { str(',') >> space? }
|
14
|
+
rule(:digit) { match('[0-9]') }
|
15
|
+
rule(:space) { match('\s').repeat(1) }
|
16
|
+
rule(:space?) { space.maybe }
|
17
|
+
|
18
|
+
# Numbers
|
19
|
+
rule(:integer) { str('-').maybe >> digit.repeat(1) >> space? }
|
20
|
+
rule(:float) {
|
21
|
+
str('-').maybe >> digit.repeat(1) >> str('.') >> digit.repeat(1) >> space?
|
22
|
+
}
|
23
|
+
rule(:number) { integer | float }
|
24
|
+
rule(:datetime) {
|
25
|
+
# 1979-05-27T07:32:00Z
|
26
|
+
digit.repeat(4) >> str("-") >>
|
27
|
+
digit.repeat(2) >> str("-") >>
|
28
|
+
digit.repeat(2) >> str("T") >>
|
29
|
+
digit.repeat(2) >> str(":") >>
|
30
|
+
digit.repeat(2) >> str(":") >>
|
31
|
+
digit.repeat(2) >> str("Z")
|
32
|
+
}
|
33
|
+
rule(:escaped_special) {
|
34
|
+
str("\\") >> match['0tnr"\\\\']
|
35
|
+
}
|
36
|
+
|
37
|
+
rule(:string_special) {
|
38
|
+
match['\0\t\n\r"\\\\']
|
39
|
+
}
|
40
|
+
rule(:string) {
|
41
|
+
str('"') >>
|
42
|
+
(escaped_special | string_special.absent? >> any).repeat >>
|
43
|
+
str('"')
|
44
|
+
}
|
45
|
+
|
46
|
+
# Field / value
|
47
|
+
# rule(:identifier) { match['_@a-zA-Z.'].repeat(1) }
|
48
|
+
rule(:identifier) { match('[^=\s]').repeat(1) }
|
49
|
+
rule(:wildcard) { match('[a-zA-Z0-9.*]').repeat(1) }
|
50
|
+
rule(:searchop) { match('[=]').as(:op) }
|
51
|
+
rule(:query_value) { number | string | datetime | wildcard }
|
52
|
+
|
53
|
+
# boolean operators search
|
54
|
+
rule(:concatop) { (str('OR') | str('AND')) >> space? }
|
55
|
+
rule(:operator) { match('[|]').as(:op) >> space? }
|
56
|
+
rule(:timerange) {
|
57
|
+
integer.as(:quantity) >> match('s|m|h|d|w').as(:quantifier)
|
58
|
+
}
|
59
|
+
|
60
|
+
# Grammar parts
|
61
|
+
rule(:rhs) {
|
62
|
+
regexp | subsearch | booleanop
|
63
|
+
# regexp | subsearch | integer | wildcard | booleanop
|
64
|
+
}
|
65
|
+
|
66
|
+
rule(:boolean_value) {
|
67
|
+
booleanparen | query_value
|
68
|
+
}
|
69
|
+
|
70
|
+
rule(:booleanop) {
|
71
|
+
boolean_value >> (space >> concatop >> boolean_value).repeat
|
72
|
+
}
|
73
|
+
|
74
|
+
rule(:booleanparen) {
|
75
|
+
lparen >> space? >> booleanop >> space? >> rparen
|
76
|
+
}
|
77
|
+
|
78
|
+
rule(:regexp) {
|
79
|
+
str('/') >> match('[^/]').repeat >> str('/')
|
80
|
+
}
|
81
|
+
|
82
|
+
rule(:last) {
|
83
|
+
str("last") >> space >> timerange.as(:timerange) >> (space >>
|
84
|
+
search.as(:search)).maybe
|
85
|
+
}
|
86
|
+
|
87
|
+
rule(:search) {
|
88
|
+
identifier.as(:field) >> space? >> searchop >> space? >>
|
89
|
+
rhs.as(:value) | rhs.as(:match)
|
90
|
+
}
|
91
|
+
|
92
|
+
rule(:binaryop) {
|
93
|
+
(search | paren).as(:left) >> space? >> operator >> job.as(:right)
|
94
|
+
}
|
95
|
+
|
96
|
+
rule(:subsearch) {
|
97
|
+
str('`') >> space? >> nested_search >> str('`')
|
98
|
+
}
|
99
|
+
|
100
|
+
rule(:nested_search) {
|
101
|
+
job.as(:initial_query) >> space? >> str('|') >> space? >>
|
102
|
+
match('[^`]').repeat.as(:extractors)
|
103
|
+
}
|
104
|
+
|
105
|
+
rule(:paren) {
|
106
|
+
lparen >> space? >> job >> space? >> rparen
|
107
|
+
}
|
108
|
+
|
109
|
+
rule(:job) {
|
110
|
+
last | search | binaryop | paren
|
111
|
+
}
|
112
|
+
|
113
|
+
# root :job
|
114
|
+
rule(:plunk_query) {
|
115
|
+
job >> (space >> job).repeat
|
116
|
+
}
|
117
|
+
|
118
|
+
root :plunk_query
|
7
119
|
end
|
8
|
-
|
9
|
-
# Single character rules
|
10
|
-
rule(:lparen) { str('(') >> space? }
|
11
|
-
rule(:rparen) { str(')') >> space? }
|
12
|
-
rule(:comma) { str(',') >> space? }
|
13
|
-
rule(:digit) { match('[0-9]') }
|
14
|
-
rule(:space) { match('\s').repeat(1) }
|
15
|
-
rule(:space?) { space.maybe }
|
16
|
-
|
17
|
-
# Numbers
|
18
|
-
rule(:integer) { str('-').maybe >> digit.repeat(1) >> space? }
|
19
|
-
rule(:float) {
|
20
|
-
str('-').maybe >> digit.repeat(1) >> str('.') >> digit.repeat(1) >> space?
|
21
|
-
}
|
22
|
-
rule(:number) { integer | float }
|
23
|
-
rule(:datetime) {
|
24
|
-
# 1979-05-27T07:32:00Z
|
25
|
-
digit.repeat(4) >> str("-") >>
|
26
|
-
digit.repeat(2) >> str("-") >>
|
27
|
-
digit.repeat(2) >> str("T") >>
|
28
|
-
digit.repeat(2) >> str(":") >>
|
29
|
-
digit.repeat(2) >> str(":") >>
|
30
|
-
digit.repeat(2) >> str("Z")
|
31
|
-
}
|
32
|
-
rule(:escaped_special) {
|
33
|
-
str("\\") >> match['0tnr"\\\\']
|
34
|
-
}
|
35
|
-
|
36
|
-
rule(:string_special) {
|
37
|
-
match['\0\t\n\r"\\\\']
|
38
|
-
}
|
39
|
-
rule(:string) {
|
40
|
-
str('"') >>
|
41
|
-
(escaped_special | string_special.absent? >> any).repeat >>
|
42
|
-
str('"')
|
43
|
-
}
|
44
|
-
|
45
|
-
# Field / value
|
46
|
-
# rule(:identifier) { match['_@a-zA-Z.'].repeat(1) }
|
47
|
-
rule(:identifier) { match('[^=\s]').repeat(1) }
|
48
|
-
rule(:wildcard) { match('[a-zA-Z0-9.*]').repeat(1) }
|
49
|
-
rule(:searchop) { match('[=]').as(:op) }
|
50
|
-
rule(:query_value) { number | string | datetime | wildcard }
|
51
|
-
|
52
|
-
# boolean operators search
|
53
|
-
rule(:concatop) { (str('OR') | str('AND')) >> space? }
|
54
|
-
rule(:operator) { match('[|]').as(:op) >> space? }
|
55
|
-
rule(:timerange) {
|
56
|
-
integer.as(:quantity) >> match('s|m|h|d|w').as(:quantifier)
|
57
|
-
}
|
58
|
-
|
59
|
-
# Grammar parts
|
60
|
-
rule(:rhs) {
|
61
|
-
regexp | subsearch | booleanop
|
62
|
-
# regexp | subsearch | integer | wildcard | booleanop
|
63
|
-
}
|
64
|
-
|
65
|
-
rule(:boolean_value) {
|
66
|
-
booleanparen | query_value
|
67
|
-
}
|
68
|
-
|
69
|
-
rule(:booleanop) {
|
70
|
-
boolean_value >> (space >> concatop >> boolean_value).repeat
|
71
|
-
}
|
72
|
-
|
73
|
-
rule(:booleanparen) {
|
74
|
-
lparen >> space? >> booleanop >> space? >> rparen
|
75
|
-
}
|
76
|
-
|
77
|
-
rule(:regexp) {
|
78
|
-
str('/') >> match('[^/]').repeat >> str('/')
|
79
|
-
}
|
80
|
-
|
81
|
-
rule(:last) {
|
82
|
-
str("last") >> space >> timerange.as(:timerange) >> (space >>
|
83
|
-
search.as(:search)).maybe
|
84
|
-
}
|
85
|
-
|
86
|
-
rule(:search) {
|
87
|
-
identifier.as(:field) >> space? >> searchop >> space? >>
|
88
|
-
rhs.as(:value) | rhs.as(:match)
|
89
|
-
}
|
90
|
-
|
91
|
-
rule(:binaryop) {
|
92
|
-
(search | paren).as(:left) >> space? >> operator >> job.as(:right)
|
93
|
-
}
|
94
|
-
|
95
|
-
rule(:subsearch) {
|
96
|
-
str('`') >> space? >> nested_search >> str('`')
|
97
|
-
}
|
98
|
-
|
99
|
-
rule(:nested_search) {
|
100
|
-
job.as(:initial_query) >> space? >> str('|') >> space? >>
|
101
|
-
match('[^`]').repeat.as(:extractors)
|
102
|
-
}
|
103
|
-
|
104
|
-
rule(:paren) {
|
105
|
-
lparen >> space? >> job >> space? >> rparen
|
106
|
-
}
|
107
|
-
|
108
|
-
rule(:job) {
|
109
|
-
last | search | binaryop | paren
|
110
|
-
}
|
111
|
-
|
112
|
-
# root :job
|
113
|
-
rule(:plunk_query) {
|
114
|
-
job >> (space >> job).repeat
|
115
|
-
}
|
116
|
-
|
117
|
-
root :plunk_query
|
118
120
|
end
|
119
|
-
|
data/lib/plunk/result_set.rb
CHANGED
@@ -1,36 +1,42 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module Plunk
|
2
|
+
class ResultSet
|
3
|
+
attr_accessor :query, :query_string
|
3
4
|
|
4
|
-
|
5
|
-
|
5
|
+
def self.configure(&block)
|
6
|
+
class_eval(&block)
|
7
|
+
end
|
8
|
+
|
9
|
+
def initialize(opts={})
|
10
|
+
@query = { query: { filtered: {}}}
|
6
11
|
|
7
|
-
|
8
|
-
|
9
|
-
@query[:query][:filtered][:query] = {
|
10
|
-
query_string: {
|
11
|
-
query: opts[:query_string] }}
|
12
|
-
@query[:query][:filtered][:filter] = {
|
13
|
-
and: [
|
14
|
-
range: {
|
15
|
-
'@timestamp' => {
|
16
|
-
gte: opts[:start_time],
|
17
|
-
lte: opts[:end_time] }}]}
|
18
|
-
else
|
19
|
-
if @query_string = opts[:query_string]
|
12
|
+
if opts.size >= 3 # use "and" filter to AND filters
|
13
|
+
@query_string = opts[:query_string]
|
20
14
|
@query[:query][:filtered][:query] = {
|
21
15
|
query_string: {
|
22
16
|
query: opts[:query_string] }}
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
17
|
+
@query[:query][:filtered][:filter] = {
|
18
|
+
and: [
|
19
|
+
range: {
|
20
|
+
'@timestamp' => {
|
21
|
+
gte: opts[:start_time],
|
22
|
+
lte: opts[:end_time] }}]}
|
23
|
+
else
|
24
|
+
if @query_string = opts[:query_string]
|
25
|
+
@query[:query][:filtered][:query] = {
|
26
|
+
query_string: {
|
27
|
+
query: opts[:query_string] }}
|
28
|
+
elsif opts[:start_time] and opts[:end_time]
|
29
|
+
@query[:query][:filtered][:query] = {
|
30
|
+
range: {
|
31
|
+
'@timestamp' => {
|
32
|
+
gte: opts[:start_time],
|
33
|
+
lte: opts[:end_time] }}}
|
34
|
+
end
|
29
35
|
end
|
30
36
|
end
|
31
|
-
end
|
32
37
|
|
33
|
-
|
34
|
-
|
38
|
+
def eval
|
39
|
+
Plunk.elasticsearch_client.search(body: @query.to_json).to_json if @query
|
40
|
+
end
|
35
41
|
end
|
36
42
|
end
|
data/lib/plunk/transformer.rb
CHANGED
@@ -1,145 +1,145 @@
|
|
1
1
|
require 'parslet'
|
2
2
|
require 'active_support/core_ext'
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
4
|
+
module Plunk
|
5
|
+
class Transformer < Parslet::Transform
|
6
|
+
|
7
|
+
rule(
|
8
|
+
field: simple(:field),
|
9
|
+
value: {
|
10
|
+
initial_query: subtree(:initial_query),
|
11
|
+
extractors: simple(:extractors)
|
12
|
+
},
|
13
|
+
op: '=',
|
14
|
+
timerange: {
|
15
|
+
quantity: simple(:quantity),
|
16
|
+
quantifier: simple(:quantifier)
|
17
|
+
}) do
|
18
|
+
|
19
|
+
int_quantity = quantity.to_s.to_i
|
20
|
+
|
21
|
+
start_time =
|
22
|
+
case quantifier
|
23
|
+
when 's'
|
24
|
+
int_quantity.seconds.ago
|
25
|
+
when 'm'
|
26
|
+
int_quantity.minutes.ago
|
27
|
+
when 'h'
|
28
|
+
int_quantity.hours.ago
|
29
|
+
when 'd'
|
30
|
+
int_quantity.days.ago
|
31
|
+
when 'w'
|
32
|
+
int_quantity.weeks.ago
|
33
|
+
end
|
34
|
+
|
35
|
+
end_time = Time.now
|
36
|
+
|
37
|
+
# recursively apply nested query
|
38
|
+
result_set = Plunk::Transformer.new.apply(initial_query)
|
39
|
+
|
40
|
+
json = JSON.parse result_set.eval
|
41
|
+
values = Plunk::Utils.extract_values json, extractors.to_s.split(',')
|
42
|
+
|
43
|
+
if values.empty?
|
44
|
+
ResultSet.new(
|
45
|
+
start_time: start_time.utc.to_datetime.iso8601(3),
|
46
|
+
end_time: end_time.utc.to_datetime.iso8601(3))
|
47
|
+
|
48
|
+
else
|
49
|
+
ResultSet.new(
|
50
|
+
query_string: "#{field}:(#{values.uniq.join(' OR ')})",
|
51
|
+
start_time: start_time.utc.to_datetime.iso8601(3),
|
52
|
+
end_time: end_time.utc.to_datetime.iso8601(3))
|
32
53
|
end
|
33
|
-
|
34
|
-
end_time = Time.now
|
35
|
-
|
36
|
-
|
37
|
-
# recursively apply nested query
|
38
|
-
result_set = Plunk::Transformer.new.apply(initial_query)
|
39
|
-
|
40
|
-
json = JSON.parse result_set.eval
|
41
|
-
values = Plunk::Elasticsearch.extract_values json, extractors.to_s.split(',')
|
42
|
-
|
43
|
-
if values.empty?
|
44
|
-
Plunk::ResultSet.new(
|
45
|
-
start_time: start_time.utc.to_datetime.iso8601(3),
|
46
|
-
end_time: end_time.utc.to_datetime.iso8601(3))
|
47
|
-
|
48
|
-
else
|
49
|
-
Plunk::ResultSet.new(
|
50
|
-
query_string: "#{field}:(#{values.uniq.join(' OR ')})",
|
51
|
-
start_time: start_time.utc.to_datetime.iso8601(3),
|
52
|
-
end_time: end_time.utc.to_datetime.iso8601(3))
|
53
54
|
end
|
54
|
-
end
|
55
|
-
|
56
|
-
rule(match: simple(:value)) do
|
57
|
-
Plunk::ResultSet.new(query_string: "#{value}")
|
58
|
-
end
|
59
|
-
|
60
|
-
rule(
|
61
|
-
field: simple(:field),
|
62
|
-
value: {
|
63
|
-
initial_query: subtree(:initial_query),
|
64
|
-
extractors: simple(:extractors)
|
65
|
-
},
|
66
|
-
op: '=') do
|
67
|
-
|
68
|
-
# recursively apply nested query
|
69
|
-
result_set = Plunk::Transformer.new.apply(initial_query)
|
70
|
-
|
71
|
-
json = JSON.parse result_set.eval
|
72
|
-
values = Plunk::Elasticsearch.extract_values json, extractors.to_s.split(',')
|
73
55
|
|
74
|
-
|
75
|
-
|
76
|
-
else
|
77
|
-
Plunk::ResultSet.new(query_string: "#{field}:(#{values.uniq.join(' OR ')})")
|
56
|
+
rule(match: simple(:value)) do
|
57
|
+
ResultSet.new(query_string: "#{value}")
|
78
58
|
end
|
79
|
-
end
|
80
|
-
|
81
|
-
rule(field: simple(:field), value: simple(:value), op: '=') do
|
82
|
-
Plunk::ResultSet.new(query_string: "#{field}:#{value}")
|
83
|
-
end
|
84
59
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
start_time =
|
94
|
-
case quantifier
|
95
|
-
when 's'
|
96
|
-
int_quantity.seconds.ago
|
97
|
-
when 'm'
|
98
|
-
int_quantity.minutes.ago
|
99
|
-
when 'h'
|
100
|
-
int_quantity.hours.ago
|
101
|
-
when 'd'
|
102
|
-
int_quantity.days.ago
|
103
|
-
when 'w'
|
104
|
-
int_quantity.weeks.ago
|
105
|
-
end
|
60
|
+
rule(
|
61
|
+
field: simple(:field),
|
62
|
+
value: {
|
63
|
+
initial_query: subtree(:initial_query),
|
64
|
+
extractors: simple(:extractors)
|
65
|
+
},
|
66
|
+
op: '=') do
|
106
67
|
|
107
|
-
|
68
|
+
# recursively apply nested query
|
69
|
+
result_set = Transformer.new.apply(initial_query)
|
108
70
|
|
109
|
-
|
110
|
-
|
111
|
-
end_time: end_time.utc.to_datetime.iso8601(3))
|
112
|
-
end
|
71
|
+
json = JSON.parse result_set.eval
|
72
|
+
values = Utils.extract_values json, extractors.to_s.split(',')
|
113
73
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
quantifier: simple(:quantifier)
|
119
|
-
}) do
|
120
|
-
|
121
|
-
int_quantity = quantity.to_s.to_i
|
122
|
-
|
123
|
-
start_time =
|
124
|
-
case quantifier
|
125
|
-
when 's'
|
126
|
-
int_quantity.seconds.ago
|
127
|
-
when 'm'
|
128
|
-
int_quantity.minutes.ago
|
129
|
-
when 'h'
|
130
|
-
int_quantity.hours.ago
|
131
|
-
when 'd'
|
132
|
-
int_quantity.days.ago
|
133
|
-
when 'w'
|
134
|
-
int_quantity.weeks.ago
|
74
|
+
if values.empty?
|
75
|
+
ResultSet.new
|
76
|
+
else
|
77
|
+
ResultSet.new(query_string: "#{field}:(#{values.uniq.join(' OR ')})")
|
135
78
|
end
|
79
|
+
end
|
136
80
|
|
137
|
-
|
81
|
+
rule(field: simple(:field), value: simple(:value), op: '=') do
|
82
|
+
ResultSet.new(query_string: "#{field}:#{value}")
|
83
|
+
end
|
138
84
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
85
|
+
rule(
|
86
|
+
timerange: {
|
87
|
+
quantity: simple(:quantity),
|
88
|
+
quantifier: simple(:quantifier)
|
89
|
+
}) do
|
90
|
+
|
91
|
+
int_quantity = quantity.to_s.to_i
|
92
|
+
|
93
|
+
start_time =
|
94
|
+
case quantifier
|
95
|
+
when 's'
|
96
|
+
int_quantity.seconds.ago
|
97
|
+
when 'm'
|
98
|
+
int_quantity.minutes.ago
|
99
|
+
when 'h'
|
100
|
+
int_quantity.hours.ago
|
101
|
+
when 'd'
|
102
|
+
int_quantity.days.ago
|
103
|
+
when 'w'
|
104
|
+
int_quantity.weeks.ago
|
105
|
+
end
|
106
|
+
|
107
|
+
end_time = Time.now
|
108
|
+
|
109
|
+
ResultSet.new(
|
110
|
+
start_time: start_time.utc.to_datetime.iso8601(3),
|
111
|
+
end_time: end_time.utc.to_datetime.iso8601(3))
|
112
|
+
end
|
113
|
+
|
114
|
+
rule(
|
115
|
+
search: simple(:result_set),
|
116
|
+
timerange: {
|
117
|
+
quantity: simple(:quantity),
|
118
|
+
quantifier: simple(:quantifier)
|
119
|
+
}) do
|
120
|
+
|
121
|
+
int_quantity = quantity.to_s.to_i
|
122
|
+
|
123
|
+
start_time =
|
124
|
+
case quantifier
|
125
|
+
when 's'
|
126
|
+
int_quantity.seconds.ago
|
127
|
+
when 'm'
|
128
|
+
int_quantity.minutes.ago
|
129
|
+
when 'h'
|
130
|
+
int_quantity.hours.ago
|
131
|
+
when 'd'
|
132
|
+
int_quantity.days.ago
|
133
|
+
when 'w'
|
134
|
+
int_quantity.weeks.ago
|
135
|
+
end
|
136
|
+
|
137
|
+
end_time = Time.now
|
138
|
+
|
139
|
+
ResultSet.new(
|
140
|
+
query_string: result_set.query_string,
|
141
|
+
start_time: start_time.utc.to_datetime.iso8601(3),
|
142
|
+
end_time: end_time.utc.to_datetime.iso8601(3))
|
143
|
+
end
|
143
144
|
end
|
144
145
|
end
|
145
|
-
|
data/lib/plunk/utils.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
module Plunk
|
2
|
+
class Utils
|
3
|
+
# nested field matcher
|
4
|
+
def self.extract_values(hash, keys)
|
5
|
+
@vals ||= []
|
6
|
+
|
7
|
+
hash.each_pair do |k, v|
|
8
|
+
if v.is_a? Hash
|
9
|
+
extract_values(v, keys)
|
10
|
+
elsif v.is_a? Array
|
11
|
+
v.flatten!
|
12
|
+
if v.first.is_a? Hash
|
13
|
+
v.each { |el| extract_values(el, keys) }
|
14
|
+
elsif keys.include? k
|
15
|
+
@vals += v
|
16
|
+
end
|
17
|
+
elsif keys.include? k
|
18
|
+
@vals << v
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
return @vals
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/plunk.gemspec
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = "plunk"
|
3
|
-
s.version = "0.
|
3
|
+
s.version = "0.2.0"
|
4
4
|
s.date = "2013-12-03"
|
5
5
|
s.add_runtime_dependency "json"
|
6
6
|
s.add_runtime_dependency "parslet"
|
7
|
-
s.add_runtime_dependency "
|
7
|
+
s.add_runtime_dependency "elasticsearch"
|
8
8
|
s.add_runtime_dependency "activesupport"
|
9
9
|
s.add_development_dependency "rspec"
|
10
10
|
s.summary = "Elasticsearch query language"
|
data/spec/spec_helper.rb
CHANGED
@@ -1,18 +1,16 @@
|
|
1
1
|
require 'rspec'
|
2
2
|
require 'plunk'
|
3
|
-
require 'plunk/parser'
|
4
|
-
require 'plunk/transformer'
|
5
|
-
require 'plunk/result_set'
|
6
|
-
require 'plunk/elasticsearch'
|
7
3
|
require 'parslet/rig/rspec'
|
8
4
|
|
9
5
|
# Print ascii_tree when exception occurs
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
6
|
+
module Plunk
|
7
|
+
class ParserWrapper < Parser
|
8
|
+
def parse(query)
|
9
|
+
begin
|
10
|
+
super(query)
|
11
|
+
rescue Parslet::ParseFailed => failure
|
12
|
+
puts failure.cause.ascii_tree
|
13
|
+
end
|
16
14
|
end
|
17
15
|
end
|
18
16
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: plunk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ram Mehta
|
@@ -40,7 +40,7 @@ dependencies:
|
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: '0'
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
|
-
name:
|
43
|
+
name: elasticsearch
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - '>='
|
@@ -97,15 +97,14 @@ files:
|
|
97
97
|
- README.md
|
98
98
|
- Rakefile
|
99
99
|
- lib/plunk.rb
|
100
|
-
- lib/plunk/elasticsearch.rb
|
101
100
|
- lib/plunk/parser.rb
|
102
101
|
- lib/plunk/result_set.rb
|
103
102
|
- lib/plunk/transformer.rb
|
103
|
+
- lib/plunk/utils.rb
|
104
104
|
- plunk.gemspec
|
105
105
|
- spec/basic_spec.rb
|
106
106
|
- spec/boolean_spec.rb
|
107
107
|
- spec/chained_search_spec.rb
|
108
|
-
- spec/elasticseach_spec.rb
|
109
108
|
- spec/field_value_spec.rb
|
110
109
|
- spec/last_spec.rb
|
111
110
|
- spec/nested_search_spec.rb
|
data/lib/plunk/elasticsearch.rb
DELETED
@@ -1,95 +0,0 @@
|
|
1
|
-
# TODO: refactor this to speed up
|
2
|
-
|
3
|
-
require 'json'
|
4
|
-
require 'rest-client'
|
5
|
-
|
6
|
-
# Wrapper for Elasticsearch API
|
7
|
-
class Plunk::Elasticsearch
|
8
|
-
attr_accessor :host, :port, :fields
|
9
|
-
|
10
|
-
def initialize(opts={})
|
11
|
-
@scheme = opts[:scheme] || "http"
|
12
|
-
@host = opts[:host] || "localhost"
|
13
|
-
@port = opts[:port] || "9200"
|
14
|
-
@size = opts[:size] || "10000"
|
15
|
-
@endpoint = "#{@scheme}://#{@host}:#{@port}"
|
16
|
-
end
|
17
|
-
|
18
|
-
# Get list of all field mappings stored in ES
|
19
|
-
# TODO: cache response from ES
|
20
|
-
def available_fields
|
21
|
-
uri = URI.escape "#{@endpoint}/_mapping"
|
22
|
-
result = JSON.parse(RestClient.get(uri))
|
23
|
-
|
24
|
-
@fields = {}
|
25
|
-
nested_values(result, 'properties')
|
26
|
-
|
27
|
-
@fields
|
28
|
-
end
|
29
|
-
|
30
|
-
def active_record_errors_for(query)
|
31
|
-
return " can't be blank." if query.blank?
|
32
|
-
|
33
|
-
uri = URI.escape "#{@endpoint}/_validate/query?explain=true"
|
34
|
-
response = RestClient.post(uri, build_ES_validator(query))
|
35
|
-
|
36
|
-
json = JSON.parse(response)
|
37
|
-
|
38
|
-
json['valid'] ? nil : json['explanations'].collect { |exp| exp['error'] }
|
39
|
-
end
|
40
|
-
|
41
|
-
#
|
42
|
-
# UTILITY METHODS
|
43
|
-
#
|
44
|
-
def search(query)
|
45
|
-
uri = URI.escape "#{@endpoint}/_search?size=#{@size}"
|
46
|
-
|
47
|
-
RestClient.post uri, query
|
48
|
-
end
|
49
|
-
|
50
|
-
# returns all values for all occurences of the given nested key
|
51
|
-
def nested_values(hash, key)
|
52
|
-
hash.each do |k, v|
|
53
|
-
if k == key
|
54
|
-
@fields.merge! v
|
55
|
-
else
|
56
|
-
nested_values(v, key) if v.is_a? Hash
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
# nested field matcher
|
62
|
-
def self.extract_values(hash, keys)
|
63
|
-
@vals ||= []
|
64
|
-
|
65
|
-
hash.each_pair do |k, v|
|
66
|
-
if v.is_a? Hash
|
67
|
-
extract_values(v, keys)
|
68
|
-
elsif v.is_a? Array
|
69
|
-
v.flatten!
|
70
|
-
if v.first.is_a? Hash
|
71
|
-
v.each { |el| extract_values(el, keys) }
|
72
|
-
elsif keys.include? k
|
73
|
-
@vals += v
|
74
|
-
end
|
75
|
-
elsif keys.include? k
|
76
|
-
@vals << v
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
|
-
return @vals
|
81
|
-
end
|
82
|
-
|
83
|
-
def build_ES_validator(query)
|
84
|
-
{
|
85
|
-
filtered: {
|
86
|
-
query: {
|
87
|
-
query_string: {
|
88
|
-
query: query
|
89
|
-
}
|
90
|
-
}
|
91
|
-
}
|
92
|
-
}.to_json
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|