sparkql 1.0.3 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/CHANGELOG.md +4 -0
- data/VERSION +1 -1
- data/lib/sparkql/evaluator.rb +152 -0
- data/lib/sparkql/expression_resolver.rb +11 -0
- data/lib/sparkql/parser_tools.rb +5 -4
- data/lib/sparkql.rb +2 -0
- data/test/support/boolean_or_bust_expression_resolver.rb +12 -0
- data/test/unit/evaluator_test.rb +84 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
Y2ExNzBkYzg0N2RiZmFkOWRlMGI2NTg0NzJiOGQ5NzNlNTA2OTA5Mw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YmYyYTBkZGQxMTFkNGZhNDg2MTA1NmZjN2M4YjcyZTZiMWQ0Njk5Ng==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
Mzk4MmY3OTE2MzNiYmEyOTg3NGMzOTdhZTRjY2NhZjhmZjgxMzE1NTJkM2Vj
|
10
|
+
OWIxZGU0YTBjNTI3M2ZhZjUxNjdlOTY4MTgzMDRmNjhiOWE3MjEwNTU4YWU0
|
11
|
+
NmQ4MDYwZDRkMDIxYzkyNWI1MjQxMThhYTFiNGFmOTZmNzA1MjI=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZGEyMzQ1YzU2NDU5YjNhNmVkOTEwNjBlYTI2MWIxMmRiY2E3NWU1MjU3Nzg1
|
14
|
+
OTgxMzc3MTc3ZGI5Njc1NWZmNGJkY2FkMWNhZDdkNmY4NThkOWUyZmUzMWRj
|
15
|
+
NDViYmQzNzEzNjBkOThiMmViYjQ4YTRhNjBhYmVlYzE1NWQwMDg=
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
v1.1.0, 2016-07-28 ([changes](https://github.com/sparkapi/sparkql/compare/v1.0.3...v1.1.0))
|
2
|
+
-------------------
|
3
|
+
* [IMPROVEMENT] Evaluation class for sparkql boolean algebra processing
|
4
|
+
|
1
5
|
v1.0.3, 2016-06-06 ([changes](https://github.com/sparkapi/sparkql/compare/v1.0.2...v1.0.3))
|
2
6
|
-------------------
|
3
7
|
* [IMPROVEMENT] Expression limit lifted to 75 expressions
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0
|
1
|
+
1.1.0
|
@@ -0,0 +1,152 @@
|
|
1
|
+
# Using an instance of ExpressionResolver to resolve the individual expressions,
|
2
|
+
# this class will evaluate the rest of a parsed sparkql string to true or false.
|
3
|
+
# Namely, this class will handle all the nesting, boolean algebra, and dropped
|
4
|
+
# fields. Plus, it has some optimizations built in to skip the processing for
|
5
|
+
# any expressions that don't contribute to the net result of the filter.
|
6
|
+
class Sparkql::Evaluator
|
7
|
+
|
8
|
+
attr_reader :processed_count
|
9
|
+
|
10
|
+
def initialize expression_resolver
|
11
|
+
@resolver = expression_resolver
|
12
|
+
end
|
13
|
+
|
14
|
+
def evaluate(expressions)
|
15
|
+
@processed_count = 0
|
16
|
+
@index = {
|
17
|
+
level: 0,
|
18
|
+
block_group: 0,
|
19
|
+
conjunction: "And",
|
20
|
+
conjunction_level: 0,
|
21
|
+
match: true,
|
22
|
+
good_ors: false,
|
23
|
+
expressions: 0
|
24
|
+
}
|
25
|
+
@groups = [@index]
|
26
|
+
expressions.each do |expression|
|
27
|
+
handle_group(expression)
|
28
|
+
adjust_expression_for_dropped_field(expression)
|
29
|
+
check_for_good_ors(expression)
|
30
|
+
next if skip?(expression)
|
31
|
+
evaluate_expression(expression)
|
32
|
+
end
|
33
|
+
cleanup
|
34
|
+
return @index[:match]
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
# prepare the group stack for the next expression
|
40
|
+
def handle_group(expression)
|
41
|
+
if @index[:block_group] == expression[:block_group]
|
42
|
+
# Noop
|
43
|
+
elsif @index[:block_group] < expression[:block_group]
|
44
|
+
@index = new_group(expression)
|
45
|
+
@groups.push(@index)
|
46
|
+
else
|
47
|
+
# Turn the group into an expression, resolve down to previous group(s)
|
48
|
+
smoosh_group(expression)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Here's the real meat. We use an internal stack to represent the result of
|
53
|
+
# each block_group. This logic is re-used when merging the final result of one
|
54
|
+
# block group with the previous.
|
55
|
+
def evaluate_expression(expression)
|
56
|
+
@processed_count += 1
|
57
|
+
evaluate_node(expression, @resolver.resolve(expression))
|
58
|
+
end
|
59
|
+
def evaluate_node(node, result)
|
60
|
+
if result == :drop
|
61
|
+
@dropped_expression = node
|
62
|
+
return result
|
63
|
+
end
|
64
|
+
if node[:unary] == "Not"
|
65
|
+
result = !result
|
66
|
+
end
|
67
|
+
if node[:conjunction] == 'Not' &&
|
68
|
+
(node[:conjunction_level] == node[:level] ||
|
69
|
+
node[:conjunction_level] == @index[:level])
|
70
|
+
@index[:match] = !result
|
71
|
+
elsif node[:conjunction] == 'And' || @index[:expressions] == 0
|
72
|
+
@index[:match] = result if @index[:match]
|
73
|
+
elsif node[:conjunction] == 'Or' && result
|
74
|
+
@index[:match] = result
|
75
|
+
end
|
76
|
+
@index[:expressions] += 1
|
77
|
+
result
|
78
|
+
end
|
79
|
+
|
80
|
+
# Optimization logic, once we find any set of And'd expressions that pass and
|
81
|
+
# run into an Or at the same level, we can skip further processing at that
|
82
|
+
# level.
|
83
|
+
def check_for_good_ors(expression)
|
84
|
+
if expression[:conjunction] == 'Or'
|
85
|
+
good_index = @index
|
86
|
+
unless expression[:conjunction_level] == @index[:level]
|
87
|
+
good_index = nil
|
88
|
+
# Well crap, now we need to go back and find that level by hand
|
89
|
+
@groups.reverse_each do |i|
|
90
|
+
if i[:level] == expression[:conjunction_level]
|
91
|
+
good_index = i
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
if !good_index.nil? && good_index[:expressions] > 0 && good_index[:match]
|
96
|
+
good_index[:good_ors] = true
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# We can skip further expression processing when And-d with a false expression
|
102
|
+
# or a "good Or" was already encountered.
|
103
|
+
def skip?(expression)
|
104
|
+
@index[:good_ors] ||
|
105
|
+
!@index[:match] && expression[:conjunction] == 'And'
|
106
|
+
end
|
107
|
+
|
108
|
+
def new_group(expression)
|
109
|
+
{
|
110
|
+
level: expression[:level],
|
111
|
+
block_group: expression[:block_group],
|
112
|
+
conjunction: expression[:conjunction],
|
113
|
+
conjunction_level: expression[:conjunction_level],
|
114
|
+
match: true,
|
115
|
+
good_ors: false,
|
116
|
+
expressions: 0
|
117
|
+
}
|
118
|
+
end
|
119
|
+
|
120
|
+
# When the last expression was dropped, we need to repair the filter by
|
121
|
+
# stealing the conjunction of that dropped field.
|
122
|
+
def adjust_expression_for_dropped_field(expression)
|
123
|
+
if @dropped_expression.nil?
|
124
|
+
return
|
125
|
+
elsif @dropped_expression[:block_group] == expression[:block_group]
|
126
|
+
expression[:conjunction] = @dropped_expression[:conjunction]
|
127
|
+
expression[:conjunction_level] = @dropped_expression[:conjunction_level]
|
128
|
+
end
|
129
|
+
@dropped_expression = nil
|
130
|
+
end
|
131
|
+
|
132
|
+
# This is similar to the cleanup step, but happens when we return from a
|
133
|
+
# nesting level. Before we can proceed, we need wrap up the result of the
|
134
|
+
# nested group.
|
135
|
+
def smoosh_group(expression)
|
136
|
+
until @groups.last[:block_group] == expression[:block_group]
|
137
|
+
last = @groups.pop
|
138
|
+
@index = @groups.last
|
139
|
+
evaluate_node(last, last[:match])
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
# pop off the group stack, evaluating each group with the previous as we go.
|
144
|
+
def cleanup
|
145
|
+
while @groups.size > 1
|
146
|
+
last = @groups.pop
|
147
|
+
@index = @groups.last
|
148
|
+
evaluate_node(last, last[:match])
|
149
|
+
end
|
150
|
+
@groups.last[:match]
|
151
|
+
end
|
152
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# Base class for handling expression resolution
|
2
|
+
class Sparkql::ExpressionResolver
|
3
|
+
|
4
|
+
VALID_RESULTS = [true, false, :drop]
|
5
|
+
|
6
|
+
# Evaluate the result of this expression. Allows for any of the values in
|
7
|
+
# VALID_RESULTS
|
8
|
+
def resolve(expression)
|
9
|
+
true
|
10
|
+
end
|
11
|
+
end
|
data/lib/sparkql/parser_tools.rb
CHANGED
@@ -60,16 +60,17 @@ module Sparkql::ParserTools
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def tokenize_unary_conjunction(conj, exp)
|
63
|
-
|
64
63
|
# Handles the case when a SparkQL filter string
|
65
64
|
# begins with a unary operator, and is nested, such as:
|
66
|
-
#
|
65
|
+
# Not (Not Field Eq 1)
|
66
|
+
# In this instance we treat the outer unary as a conjunction.
|
67
67
|
if @expression_count == 1 && @lexer.level > 0
|
68
|
-
exp.first[:conjunction] = conj
|
68
|
+
exp.first[:conjunction] = conj
|
69
|
+
exp.first[:conjunction_level] = @lexer.level - 1
|
69
70
|
end
|
70
|
-
|
71
71
|
exp.first[:unary] = conj
|
72
72
|
exp.first[:unary_level] = @lexer.level
|
73
|
+
|
73
74
|
exp
|
74
75
|
end
|
75
76
|
|
data/lib/sparkql.rb
CHANGED
@@ -2,6 +2,8 @@ require "sparkql/version"
|
|
2
2
|
require "sparkql/token"
|
3
3
|
require "sparkql/errors"
|
4
4
|
require "sparkql/expression_state"
|
5
|
+
require "sparkql/expression_resolver"
|
6
|
+
require "sparkql/evaluator"
|
5
7
|
require "sparkql/lexer"
|
6
8
|
require "sparkql/function_resolver"
|
7
9
|
require "sparkql/parser_tools"
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# A super simple expression resolver for testing... returns the boolean value as
|
2
|
+
# the result for the expression, or when not a boolean, drops the expression.
|
3
|
+
class BooleanOrBustExpressionResolver < Sparkql::ExpressionResolver
|
4
|
+
|
5
|
+
def resolve(expression)
|
6
|
+
if expression[:type] == :boolean
|
7
|
+
"true" == expression[:value]
|
8
|
+
else
|
9
|
+
:drop
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
require 'support/boolean_or_bust_expression_resolver'
|
3
|
+
|
4
|
+
class EvaluatorTest < Test::Unit::TestCase
|
5
|
+
include Sparkql
|
6
|
+
|
7
|
+
def test_simple
|
8
|
+
assert sample('Test Eq true')
|
9
|
+
assert !sample('Test Eq false')
|
10
|
+
assert sample("Test Eq 'Drop'")
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_conjunction
|
14
|
+
assert sample('Test Eq true And Test Eq true')
|
15
|
+
assert !sample('Test Eq false And Test Eq true')
|
16
|
+
assert !sample('Test Eq false And Test Eq false')
|
17
|
+
# Ors
|
18
|
+
assert sample("Test Eq true Or Test Eq true")
|
19
|
+
assert sample("Test Eq true Or Test Eq false")
|
20
|
+
assert sample("Test Eq false Or Test Eq true")
|
21
|
+
assert !sample("Test Eq false Or Test Eq false")
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_dropped_field_handling
|
25
|
+
assert sample("Test Eq 'Drop' And Test Eq true")
|
26
|
+
assert !sample("Test Eq 'Drop' And Test Eq false")
|
27
|
+
assert !sample("Test Eq 'Drop' Or Test Eq false")
|
28
|
+
assert sample("Test Eq 'Drop' Or Test Eq true")
|
29
|
+
assert sample("Test Eq false And Test Eq 'Drop' Or Test Eq true")
|
30
|
+
assert sample("Test Eq false Or (Test Eq 'Drop' And Test Eq true)")
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_nesting
|
34
|
+
assert sample("Test Eq true Or (Test Eq true) And Test Eq false And (Test Eq true)")
|
35
|
+
assert sample("Test Eq true Or ((Test Eq false) And Test Eq false) And (Test Eq false)")
|
36
|
+
assert sample("(Test Eq false Or Test Eq true) Or (Test Eq false Or Test Eq false)")
|
37
|
+
assert sample("(Test Eq true And Test Eq true) Or (Test Eq false)")
|
38
|
+
assert sample("(Test Eq true And Test Eq true) Or (Test Eq false And Test Eq true)")
|
39
|
+
assert !sample("(Test Eq false And Test Eq true) Or (Test Eq false)")
|
40
|
+
assert sample("Test Eq true And ((Test Eq true And Test Eq false) Or Test Eq true)")
|
41
|
+
assert !sample("Test Eq true And ((Test Eq true And Test Eq false) Or Test Eq false) And Test Eq true")
|
42
|
+
assert !sample("Test Eq true And ((Test Eq true And Test Eq false) Or Test Eq false) Or Test Eq false")
|
43
|
+
assert sample("Test Eq true And ((Test Eq true And Test Eq false) Or Test Eq false) Or Test Eq true")
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_nots
|
47
|
+
assert !sample("Not Test Eq true")
|
48
|
+
assert sample("Not Test Eq false")
|
49
|
+
assert !sample("Not (Test Eq true)")
|
50
|
+
assert sample("Not (Test Eq false)")
|
51
|
+
assert sample("Test Eq true Not Test Eq false")
|
52
|
+
assert !sample("Test Eq true Not Test Eq true")
|
53
|
+
assert sample("Test Eq true Not (Test Eq false Or Test Eq false)")
|
54
|
+
assert sample("Test Eq true Not (Test Eq false And Test Eq false)")
|
55
|
+
assert !sample("Test Eq true Not (Test Eq false Or Test Eq true)")
|
56
|
+
assert !sample("Test Eq true Not (Test Eq true Or Test Eq false)")
|
57
|
+
assert !sample("Test Eq true Not (Not Test Eq false)")
|
58
|
+
assert sample("Not (Not Test Eq true)")
|
59
|
+
assert sample("Not (Not(Not Test Eq true))")
|
60
|
+
end
|
61
|
+
|
62
|
+
def test_optimizations
|
63
|
+
assert sample("Test Eq true Or Test Eq false And Test Eq false")
|
64
|
+
assert_equal 1, @evaluator.processed_count
|
65
|
+
assert sample("Test Eq false Or Test Eq true And Test Eq true")
|
66
|
+
assert_equal 3, @evaluator.processed_count
|
67
|
+
assert sample("(Test Eq true Or Test Eq false) And Test Eq true")
|
68
|
+
assert_equal 2, @evaluator.processed_count
|
69
|
+
assert sample("(Test Eq false Or Test Eq true) And Test Eq true")
|
70
|
+
assert_equal 3, @evaluator.processed_count
|
71
|
+
end
|
72
|
+
|
73
|
+
# Here's some examples from prospector's tests that have been simplified a bit.
|
74
|
+
def test_advanced
|
75
|
+
assert !sample("MlsStatus Eq false And PropertyType Eq true And (City Eq true Or City Eq false)")
|
76
|
+
end
|
77
|
+
|
78
|
+
def sample filter
|
79
|
+
@parser = Parser.new
|
80
|
+
@expressions = @parser.parse(filter)
|
81
|
+
@evaluator = Evaluator.new(BooleanOrBustExpressionResolver.new())
|
82
|
+
@evaluator.evaluate(@expressions)
|
83
|
+
end
|
84
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sparkql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wade McEwen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-07-
|
11
|
+
date: 2016-07-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: georuby
|
@@ -111,6 +111,8 @@ files:
|
|
111
111
|
- VERSION
|
112
112
|
- lib/sparkql.rb
|
113
113
|
- lib/sparkql/errors.rb
|
114
|
+
- lib/sparkql/evaluator.rb
|
115
|
+
- lib/sparkql/expression_resolver.rb
|
114
116
|
- lib/sparkql/expression_state.rb
|
115
117
|
- lib/sparkql/function_resolver.rb
|
116
118
|
- lib/sparkql/geo.rb
|
@@ -127,8 +129,10 @@ files:
|
|
127
129
|
- script/markdownify.rb
|
128
130
|
- script/release
|
129
131
|
- sparkql.gemspec
|
132
|
+
- test/support/boolean_or_bust_expression_resolver.rb
|
130
133
|
- test/test_helper.rb
|
131
134
|
- test/unit/errors_test.rb
|
135
|
+
- test/unit/evaluator_test.rb
|
132
136
|
- test/unit/expression_state_test.rb
|
133
137
|
- test/unit/function_resolver_test.rb
|
134
138
|
- test/unit/geo/record_circle_test.rb
|
@@ -160,8 +164,10 @@ signing_key:
|
|
160
164
|
specification_version: 4
|
161
165
|
summary: API Parser engine for filter searching
|
162
166
|
test_files:
|
167
|
+
- test/support/boolean_or_bust_expression_resolver.rb
|
163
168
|
- test/test_helper.rb
|
164
169
|
- test/unit/errors_test.rb
|
170
|
+
- test/unit/evaluator_test.rb
|
165
171
|
- test/unit/expression_state_test.rb
|
166
172
|
- test/unit/function_resolver_test.rb
|
167
173
|
- test/unit/geo/record_circle_test.rb
|